public FakeStorageBlobContainer(MemoryBlobStore store, string containerName, IStorageBlobClient parent) { _store = store; _containerName = containerName; _parent = parent; _uri = new Uri("http://localhost/" + containerName); }
public BlobFunctionOutputDefinition(IStorageBlobClient client, LocalBlobDescriptor outputBlob, LocalBlobDescriptor parameterLogBlob) { _client = client; _outputBlob = outputBlob; _parameterLogBlob = parameterLogBlob; }
public BlobContainerBinding(string parameterName, IArgumentBinding<IStorageBlobContainer> argumentBinding, IStorageBlobClient client, IBindableBlobPath path) { _parameterName = parameterName; _argumentBinding = argumentBinding; _client = client; _path = path; }
public SingletonManager(IStorageBlobClient blobClient, IBackgroundExceptionDispatcher backgroundExceptionDispatcher, SingletonConfiguration config, TraceWriter trace) { _backgroundExceptionDispatcher = backgroundExceptionDispatcher; _directory = blobClient.GetContainerReference(HostContainerNames.Hosts) .GetDirectoryReference(HostDirectoryNames.SingletonLocks); _config = config; _trace = trace; }
public BlobBinding(string parameterName, IBlobArgumentBinding argumentBinding, IStorageBlobClient client, IBindableBlobPath path) { _parameterName = parameterName; _argumentBinding = argumentBinding; _client = client; _accountName = BlobClient.GetAccountName(client); _path = path; _converter = CreateConverter(_client, path, argumentBinding.ValueType); }
public static string GetAccountName(IStorageBlobClient client) { if (client == null) { return null; } return StorageClient.GetAccountName(client.Credentials); }
private static IAsyncObjectToTypeConverter<IStorageBlob> CreateConverter(IStorageBlobClient client, IBindableBlobPath path, Type argumentType) { return new CompositeAsyncObjectToTypeConverter<IStorageBlob>( new BlobOutputConverter<IStorageBlob>(new AsyncConverter<IStorageBlob, IStorageBlob>( new IdentityConverter<IStorageBlob>())), new BlobOutputConverter<ICloudBlob>(new AsyncConverter<ICloudBlob, IStorageBlob>( new CloudBlobToStorageBlobConverter())), new BlobOutputConverter<string>(new StringToStorageBlobConverter(client, path, argumentType))); }
internal static bool TryConvert(object value, IStorageBlobClient client, out IStorageBlobContainer container, out BlobPath path) { container = null; path = null; string fullPath = value as string; if (fullPath != null) { path = BlobPath.ParseAndValidate(fullPath, isContainerBinding: true); container = client.GetContainerReference(path.ContainerName); return true; } return false; }
public FunctionStatusLogger(IHostIdProvider hostIdProvider, IStorageBlobClient blobClient) { if (hostIdProvider == null) { throw new ArgumentNullException("hostIdProvider"); } if (blobClient == null) { throw new ArgumentNullException("blobClient"); } _hostIdProvider = hostIdProvider; _blobClient = blobClient; _hostContainer = _blobClient.GetContainerReference(HostContainerNames.Hosts); }
public void BlobPolling_IgnoresClockSkew() { int testScanBlobLimitPerPoll = 3; string containerName = Path.GetRandomFileName(); IStorageAccount account = CreateFakeStorageAccount(); IStorageBlobClient client = account.CreateBlobClient(); var now = DateTimeOffset.UtcNow; var timeMap = new Dictionary <string, DateTimeOffset>(); IStorageBlobContainer container = new SkewableFakeStorageBlobContainer(new MemoryBlobStore(), containerName, client, (IStorageBlobResultSegment blobs) => { // Simulate some extreme clock skew -- the first one's LastUpdated // wll be 60 seconds ago and the second will be 59 seconds ago. foreach (IStorageBlob blob in blobs.Results) { ((FakeStorageBlobProperties)blob.Properties).LastModified = timeMap[blob.Name]; } }); IBlobListenerStrategy product = new ScanBlobScanLogHybridPollingStrategy(new TestBlobScanInfoManager()); LambdaBlobTriggerExecutor executor = new LambdaBlobTriggerExecutor(); typeof(ScanBlobScanLogHybridPollingStrategy) .GetField("_scanBlobLimitPerPoll", BindingFlags.Instance | BindingFlags.NonPublic) .SetValue(product, testScanBlobLimitPerPoll); product.Register(container, executor); product.Start(); List <string> expectedNames = new List <string>(); expectedNames.Add(CreateAblobAndUploadToContainer(container)); timeMap[expectedNames.Single()] = now.AddSeconds(-60); RunExecuterWithExpectedBlobs(expectedNames, product, executor); expectedNames.Clear(); expectedNames.Add(CreateAblobAndUploadToContainer(container)); timeMap[expectedNames.Single()] = now.AddSeconds(-59); // We should see the new item. RunExecuterWithExpectedBlobs(expectedNames, product, executor); }
private async Task <IStorageBlob> GetBlobAsync( BlobAttribute blobAttribute, CancellationToken cancellationToken, Type requestedType = null) { IStorageBlobClient client = await GetClientAsync(blobAttribute, cancellationToken); BlobPath boundPath = BlobPath.ParseAndValidate(blobAttribute.BlobPath); IStorageBlobContainer container = client.GetContainerReference(boundPath.ContainerName); if (blobAttribute.Access != FileAccess.Read) { await container.CreateIfNotExistsAsync(cancellationToken); } IStorageBlob blob = await container.GetBlobReferenceForArgumentTypeAsync( boundPath.BlobName, requestedType, cancellationToken); return(blob); }
public void ExecuteAsync_IfBlobDoesNotMatchPattern_ReturnsTrue() { // Arrange IStorageAccount account = CreateAccount(); IStorageBlobClient client = account.CreateBlobClient(); string containerName = "container"; IStorageBlobContainer container = client.GetContainerReference(containerName); IStorageBlobContainer otherContainer = client.GetContainerReference("other"); IBlobPathSource input = BlobPathSource.Create(containerName + "/{name}"); ITriggerExecutor <IStorageBlob> product = CreateProductUnderTest(input); IStorageBlob blob = otherContainer.GetBlockBlobReference("nonmatch"); // Act Task <bool> task = product.ExecuteAsync(blob, CancellationToken.None); // Assert Assert.True(task.Result); }
internal IStorageBlobDirectory GetLockDirectory(string accountName) { if (string.IsNullOrEmpty(accountName)) { accountName = ConnectionStringNames.Storage; } IStorageBlobDirectory storageDirectory = null; if (!_lockDirectoryMap.TryGetValue(accountName, out storageDirectory)) { Task <IStorageAccount> task = _accountProvider.GetAccountAsync(accountName, CancellationToken.None); IStorageAccount storageAccount = task.Result; IStorageBlobClient blobClient = storageAccount.CreateBlobClient(); storageDirectory = blobClient.GetContainerReference(HostContainerNames.Hosts) .GetDirectoryReference(HostDirectoryNames.SingletonLocks); _lockDirectoryMap[accountName] = storageDirectory; } return(storageDirectory); }
private IListener CreateQueueMessageToTriggerExecutionListener( ISharedContextProvider sharedContextProvider, SharedQueueWatcher sharedQueueWatcher, IStorageQueueClient queueClient, IStorageQueue hostBlobTriggerQueue, IStorageBlobClient blobClient, IBlobWrittenWatcher blobWrittenWatcher) { SharedBlobQueueListener sharedListener = sharedContextProvider.GetOrCreate <SharedBlobQueueListener>( new SharedBlobQueueListenerFactory(sharedQueueWatcher, queueClient, hostBlobTriggerQueue, _queueConfiguration, _backgroundExceptionDispatcher, _trace, blobWrittenWatcher)); BlobQueueRegistration registration = new BlobQueueRegistration { Executor = _executor, BlobClient = blobClient }; sharedListener.Register(_functionId, registration); return(new BlobListener(sharedListener)); }
public BlobTriggerBinding(ParameterInfo parameter, IStorageAccount hostAccount, IStorageAccount dataAccount, IBlobPathSource path, IHostIdProvider hostIdProvider, IQueueConfiguration queueConfiguration, JobHostBlobsConfiguration blobsConfiguration, IWebJobsExceptionHandler exceptionHandler, IContextSetter <IBlobWrittenWatcher> blobWrittenWatcherSetter, IContextSetter <IMessageEnqueuedWatcher> messageEnqueuedWatcherSetter, ISharedContextProvider sharedContextProvider, SingletonManager singletonManager, ILoggerFactory loggerFactory) { _parameter = parameter ?? throw new ArgumentNullException(nameof(parameter)); _hostAccount = hostAccount ?? throw new ArgumentNullException(nameof(hostAccount)); _dataAccount = dataAccount ?? throw new ArgumentNullException(nameof(dataAccount)); StorageClientFactoryContext context = new StorageClientFactoryContext { Parameter = parameter }; _blobClient = dataAccount.CreateBlobClient(context); _accountName = BlobClient.GetAccountName(_blobClient); _path = path ?? throw new ArgumentNullException(nameof(path)); _hostIdProvider = hostIdProvider ?? throw new ArgumentNullException(nameof(hostIdProvider)); _queueConfiguration = queueConfiguration ?? throw new ArgumentNullException(nameof(queueConfiguration)); _blobsConfiguration = blobsConfiguration ?? throw new ArgumentNullException(nameof(blobsConfiguration)); _exceptionHandler = exceptionHandler ?? throw new ArgumentNullException(nameof(exceptionHandler)); _blobWrittenWatcherSetter = blobWrittenWatcherSetter ?? throw new ArgumentNullException(nameof(blobWrittenWatcherSetter)); _messageEnqueuedWatcherSetter = messageEnqueuedWatcherSetter ?? throw new ArgumentNullException(nameof(messageEnqueuedWatcherSetter)); _sharedContextProvider = sharedContextProvider ?? throw new ArgumentNullException(nameof(sharedContextProvider)); _singletonManager = singletonManager ?? throw new ArgumentNullException(nameof(singletonManager)); _loggerFactory = loggerFactory; _converter = CreateConverter(_blobClient); _bindingDataContract = CreateBindingDataContract(path); }
public StorageBlobScanInfoManager(string hostId, IStorageBlobClient blobClient) { if (string.IsNullOrEmpty(hostId)) { throw new ArgumentNullException("hostId"); } if (blobClient == null) { throw new ArgumentNullException("blobClient"); } _hostId = hostId; JsonSerializerSettings settings = new JsonSerializerSettings { DateFormatHandling = DateFormatHandling.IsoDateFormat }; _serializer = JsonSerializer.Create(settings); string blobScanInfoDirectoryPath = string.Format(CultureInfo.InvariantCulture, "blobscaninfo/{0}", _hostId); _blobScanInfoDirectory = blobClient.GetContainerReference(HostContainerNames.Hosts).GetDirectoryReference(blobScanInfoDirectoryPath); }
internal IStorageBlobDirectory GetLockDirectory(string accountName) { if (string.IsNullOrEmpty(accountName)) { accountName = ConnectionStringNames.Storage; } IStorageBlobDirectory storageDirectory = null; if (!_lockDirectoryMap.TryGetValue(accountName, out storageDirectory)) { Task <IStorageAccount> task = _accountProvider.GetStorageAccountAsync(accountName, CancellationToken.None); IStorageAccount storageAccount = task.Result; // singleton requires block blobs, cannot be premium storageAccount.AssertTypeOneOf(StorageAccountType.GeneralPurpose, StorageAccountType.BlobOnly); IStorageBlobClient blobClient = storageAccount.CreateBlobClient(); storageDirectory = blobClient.GetContainerReference(HostContainerNames.Hosts) .GetDirectoryReference(HostDirectoryNames.SingletonLocks); _lockDirectoryMap[accountName] = storageDirectory; } return(storageDirectory); }
public async Task RegisterAsync(IStorageBlobContainer container, ITriggerExecutor <IStorageBlob> triggerExecutor, CancellationToken cancellationToken) { ThrowIfDisposed(); // Initial background scans for all containers happen on first Execute call. // Prevent accidental late registrations. // (Also prevents incorrect concurrent execution of Register with Execute.) if (_initialScanThread.ThreadState != ThreadState.Unstarted) { throw new InvalidOperationException("All registrations must be created before execution begins."); } ICollection <ITriggerExecutor <IStorageBlob> > containerRegistrations; if (_registrations.ContainsKey(container)) { containerRegistrations = _registrations[container]; } else { containerRegistrations = new List <ITriggerExecutor <IStorageBlob> >(); _registrations.Add(container, containerRegistrations); } containerRegistrations.Add(triggerExecutor); IStorageBlobClient client = container.ServiceClient; if (!_logListeners.ContainsKey(client)) { BlobLogListener logListener = await BlobLogListener.CreateAsync(client, cancellationToken); _logListeners.Add(client, logListener); } }
private static async Task ValidateCredentialsAsyncCore(IStorageAccount account, CancellationToken cancellationToken) { // Verify the credentials are correct. // Have to actually ping a storage operation. IStorageBlobClient client = account.CreateBlobClient(); try { // This can hang for a long time if the account name is wrong. // If will fail fast if the password is incorrect. await client.GetServicePropertiesAsync(cancellationToken); } catch (OperationCanceledException) { throw; } catch { string message = String.Format(CultureInfo.CurrentCulture, "The account credentials for '{0}' are incorrect.", account.Credentials.AccountName); throw new InvalidOperationException(message); } }
private async Task <IListener> CreateBlobDiscoveryToQueueMessageListenerAsync(string hostId, SharedBlobListener sharedBlobListener, IStorageBlobClient blobClient, IStorageQueue hostBlobTriggerQueue, IMessageEnqueuedWatcher messageEnqueuedWatcher, CancellationToken cancellationToken) { BlobTriggerExecutor triggerExecutor = new BlobTriggerExecutor(hostId, _functionId, _input, BlobETagReader.Instance, new BlobReceiptManager(blobClient), new BlobTriggerQueueWriter(hostBlobTriggerQueue, messageEnqueuedWatcher)); await sharedBlobListener.RegisterAsync(_container, triggerExecutor, cancellationToken); return(new BlobListener(sharedBlobListener)); }
// Test that the credentials are valid and classify the account.Type as one of StorageAccountTypes private static async Task ValidateCredentialsAsyncCore(IStorageAccount account, CancellationToken cancellationToken) { // Verify the credentials are correct. // Have to actually ping a storage operation. IStorageBlobClient client = account.CreateBlobClient(); try { // This can hang for a long time if the account name is wrong. // If will fail fast if the password is incorrect. await client.GetServicePropertiesAsync(cancellationToken); } catch (OperationCanceledException) { throw; } catch (Exception e) { var storageException = e as StorageException; if (storageException?.RequestInformation?.HttpStatusCode == 400 && storageException?.RequestInformation?.ExtendedErrorInformation?.ErrorCode == "InvalidQueryParameterValue") { // Premium storage accounts do not support the GetServicePropertiesAsync call, and respond with a 400 'InvalidQueryParameterValue'. // If we see this error response classify the account as a premium account account.Type = StorageAccountType.Premium; return; } else { // If not a recognized error, the credentials are invalid string message = String.Format(CultureInfo.CurrentCulture, "Invalid storage account '{0}'. Please make sure your credentials are correct.", account.Credentials.AccountName); throw new InvalidOperationException(message); } } IStorageQueueClient queueClient = account.CreateQueueClient(); IStorageQueue queue = queueClient.GetQueueReference("name"); try { await queue.ExistsAsync(cancellationToken); } catch (OperationCanceledException) { throw; } catch (StorageException exception) { WebException webException = exception.GetBaseException() as WebException; if (webException?.Status == WebExceptionStatus.NameResolutionFailure) { // Blob-only storage accounts do not support services other than Blob. // If we see a name resolution failure on the queue endpoint classify as a blob-only account account.Type = StorageAccountType.BlobOnly; return; } throw; } }
public BlobFunctionOutputLogger(IStorageBlobClient client) : this(client.GetContainerReference( HostContainerNames.Hosts).GetDirectoryReference(HostDirectoryNames.OutputLogs)) { }
public BlobTriggerBinding(string parameterName, IArgumentBinding <IStorageBlob> argumentBinding, IStorageAccount account, IBlobPathSource path, IHostIdProvider hostIdProvider, IQueueConfiguration queueConfiguration, IBackgroundExceptionDispatcher backgroundExceptionDispatcher, IContextSetter <IBlobWrittenWatcher> blobWrittenWatcherSetter, IContextSetter <IMessageEnqueuedWatcher> messageEnqueuedWatcherSetter, ISharedContextProvider sharedContextProvider, TextWriter log) { if (argumentBinding == null) { throw new ArgumentNullException("argumentBinding"); } if (account == null) { throw new ArgumentNullException("account"); } if (path == null) { throw new ArgumentNullException("path"); } if (hostIdProvider == null) { throw new ArgumentNullException("hostIdProvider"); } if (queueConfiguration == null) { throw new ArgumentNullException("queueConfiguration"); } if (backgroundExceptionDispatcher == null) { throw new ArgumentNullException("backgroundExceptionDispatcher"); } if (blobWrittenWatcherSetter == null) { throw new ArgumentNullException("blobWrittenWatcherSetter"); } if (messageEnqueuedWatcherSetter == null) { throw new ArgumentNullException("messageEnqueuedWatcherSetter"); } if (sharedContextProvider == null) { throw new ArgumentNullException("sharedContextProvider"); } if (log == null) { throw new ArgumentNullException("log"); } _parameterName = parameterName; _argumentBinding = argumentBinding; _account = account; _client = account.CreateBlobClient(); _accountName = BlobClient.GetAccountName(_client); _path = path; _hostIdProvider = hostIdProvider; _queueConfiguration = queueConfiguration; _backgroundExceptionDispatcher = backgroundExceptionDispatcher; _blobWrittenWatcherSetter = blobWrittenWatcherSetter; _messageEnqueuedWatcherSetter = messageEnqueuedWatcherSetter; _sharedContextProvider = sharedContextProvider; _log = log; _converter = CreateConverter(_client); _bindingDataContract = CreateBindingDataContract(path); }
public BlobTriggerBinding(ParameterInfo parameter, IArgumentBinding<IStorageBlob> argumentBinding, IStorageAccount hostAccount, IStorageAccount dataAccount, IBlobPathSource path, IHostIdProvider hostIdProvider, IQueueConfiguration queueConfiguration, IBackgroundExceptionDispatcher backgroundExceptionDispatcher, IContextSetter<IBlobWrittenWatcher> blobWrittenWatcherSetter, IContextSetter<IMessageEnqueuedWatcher> messageEnqueuedWatcherSetter, ISharedContextProvider sharedContextProvider, TraceWriter trace) { if (parameter == null) { throw new ArgumentNullException("parameter"); } if (argumentBinding == null) { throw new ArgumentNullException("argumentBinding"); } if (hostAccount == null) { throw new ArgumentNullException("hostAccount"); } if (dataAccount == null) { throw new ArgumentNullException("dataAccount"); } if (path == null) { throw new ArgumentNullException("path"); } if (hostIdProvider == null) { throw new ArgumentNullException("hostIdProvider"); } if (queueConfiguration == null) { throw new ArgumentNullException("queueConfiguration"); } if (backgroundExceptionDispatcher == null) { throw new ArgumentNullException("backgroundExceptionDispatcher"); } if (blobWrittenWatcherSetter == null) { throw new ArgumentNullException("blobWrittenWatcherSetter"); } if (messageEnqueuedWatcherSetter == null) { throw new ArgumentNullException("messageEnqueuedWatcherSetter"); } if (sharedContextProvider == null) { throw new ArgumentNullException("sharedContextProvider"); } if (trace == null) { throw new ArgumentNullException("trace"); } _parameter = parameter; _argumentBinding = argumentBinding; _hostAccount = hostAccount; _dataAccount = dataAccount; StorageClientFactoryContext context = new StorageClientFactoryContext { Parameter = parameter }; _blobClient = dataAccount.CreateBlobClient(context); _accountName = BlobClient.GetAccountName(_blobClient); _path = path; _hostIdProvider = hostIdProvider; _queueConfiguration = queueConfiguration; _backgroundExceptionDispatcher = backgroundExceptionDispatcher; _blobWrittenWatcherSetter = blobWrittenWatcherSetter; _messageEnqueuedWatcherSetter = messageEnqueuedWatcherSetter; _sharedContextProvider = sharedContextProvider; _trace = trace; _converter = CreateConverter(_blobClient); _bindingDataContract = CreateBindingDataContract(path); }
/// <summary>Initializes a new instance of the <see cref="StorageBlobContainer"/> class.</summary> /// <param name="parent">The parent blob client.</param> /// <param name="sdk">The SDK container to wrap.</param> public StorageBlobContainer(IStorageBlobClient parent, CloudBlobContainer sdk) { _parent = parent; _sdk = sdk; }
public BlobFunctionOutputLogger(IStorageBlobClient client) : this(client.GetContainerReference(HostContainerNames.Hosts).GetDirectoryReference(HostDirectoryNames.OutputLogs)) { }
public async Task <IListener> CreateAsync(CancellationToken cancellationToken) { // Note that these clients are intentionally for the storage account rather than for the dashboard account. // We use the storage, not dashboard, account for the blob receipt container and blob trigger queues. IStorageQueueClient primaryQueueClient = _hostAccount.CreateQueueClient(); IStorageBlobClient primaryBlobClient = _hostAccount.CreateBlobClient(); // Important: We're using the storage account of the function target here, which is the account that the // function the listener is for is targeting. This is the account that will be used // to read the trigger blob. IStorageBlobClient targetBlobClient = _dataAccount.CreateBlobClient(); IStorageQueueClient targetQueueClient = _dataAccount.CreateQueueClient(); string hostId = await _hostIdProvider.GetHostIdAsync(cancellationToken); string hostBlobTriggerQueueName = HostQueueNames.GetHostBlobTriggerQueueName(hostId); IStorageQueue hostBlobTriggerQueue = primaryQueueClient.GetQueueReference(hostBlobTriggerQueueName); SharedQueueWatcher sharedQueueWatcher = _sharedContextProvider.GetOrCreateInstance <SharedQueueWatcher>( new SharedQueueWatcherFactory(_messageEnqueuedWatcherSetter)); SharedBlobListener sharedBlobListener = _sharedContextProvider.GetOrCreateInstance <SharedBlobListener>( new SharedBlobListenerFactory(hostId, _hostAccount, _exceptionHandler, _blobWrittenWatcherSetter)); // Register the blob container we wish to monitor with the shared blob listener. await RegisterWithSharedBlobListenerAsync(hostId, sharedBlobListener, primaryBlobClient, hostBlobTriggerQueue, sharedQueueWatcher, cancellationToken); // Create a "bridge" listener that will monitor the blob // notification queue and dispatch to the target job function. SharedBlobQueueListener sharedBlobQueueListener = _sharedContextProvider.GetOrCreateInstance <SharedBlobQueueListener>( new SharedBlobQueueListenerFactory(_hostAccount, sharedQueueWatcher, hostBlobTriggerQueue, _queueConfiguration, _exceptionHandler, _trace, _loggerFactory, sharedBlobListener.BlobWritterWatcher)); var queueListener = new BlobListener(sharedBlobQueueListener); // determine which client to use for the poison queue // by default this should target the same storage account // as the blob container we're monitoring var poisonQueueClient = targetQueueClient; if (_dataAccount.Type != StorageAccountType.GeneralPurpose || _blobsConfiguration.CentralizedPoisonQueue) { // use the primary storage account if the centralize flag is true, // or if the target storage account doesn't support queues poisonQueueClient = primaryQueueClient; } // Register our function with the shared blob queue listener RegisterWithSharedBlobQueueListenerAsync(sharedBlobQueueListener, targetBlobClient, poisonQueueClient); // check a flag in the shared context to see if we've created the singleton // shared blob listener in this host instance object singletonListenerCreated = false; if (!_sharedContextProvider.TryGetValue(SingletonBlobListenerScopeId, out singletonListenerCreated)) { // Create a singleton shared blob listener, since we only // want a single instance of the blob poll/scan logic to be running // across host instances var singletonBlobListener = _singletonManager.CreateHostSingletonListener( new BlobListener(sharedBlobListener), SingletonBlobListenerScopeId); _sharedContextProvider.SetValue(SingletonBlobListenerScopeId, true); return(new CompositeListener(singletonBlobListener, queueListener)); } else { // We've already created the singleton blob listener // so just return our queue listener. Note that while we want the // blob scan to be singleton, the shared queue listener needs to run // on ALL instances so load can be scaled out return(queueListener); } }
// Populate the List<> with blob logs for the given prefix. // http://blogs.msdn.com/b/windowsazurestorage/archive/2011/08/03/windows-azure-storage-logging-using-logs-to-track-storage-requests.aspx private static async Task GetLogsWithPrefixAsync(List <IStorageBlob> selectedLogs, IStorageBlobClient blobClient, string prefix, CancellationToken cancellationToken) { // List the blobs using the prefix IEnumerable <IStorageListBlobItem> blobs = await blobClient.ListBlobsAsync(prefix, useFlatBlobListing : true, blobListingDetails : BlobListingDetails.Metadata, cancellationToken : cancellationToken); // iterate through each blob and figure the start and end times in the metadata // Type cast to IStorageBlob is safe due to useFlatBlobListing: true above. foreach (IStorageBlob item in blobs) { IStorageBlob log = item as IStorageBlob; if (log != null) { // we will exclude the file if the file does not have log entries in the interested time range. string logType = log.Metadata[LogType]; bool hasWrites = logType.Contains("write"); if (hasWrites) { selectedLogs.Add(log); } } } }
public BlobContainerBinding(string parameterName, IArgumentBinding <IStorageBlobContainer> argumentBinding, IStorageBlobClient client, IBindableBlobPath path) { _parameterName = parameterName; _argumentBinding = argumentBinding; _client = client; _path = path; }
public StringToStorageBlobConverter(IStorageBlobClient client, IBindableBlobPath defaultPath, Type argumentType) { _client = client; _defaultPath = defaultPath; _argumentType = argumentType; }
public SkewableFakeStorageBlobContainer(MemoryBlobStore store, string containerName, IStorageBlobClient parent, Action <IStorageBlobResultSegment> onListBlobsSegmented) : base(store, containerName, parent) { _onListBlobsSegmented = onListBlobsSegmented; }
public static async Task <IEnumerable <IStorageListBlobItem> > ListBlobsAsync(this IStorageBlobClient client, string prefix, bool useFlatBlobListing, BlobListingDetails blobListingDetails, CancellationToken cancellationToken) { if (client == null) { throw new ArgumentNullException("client"); } List <IStorageListBlobItem> allResults = new List <IStorageListBlobItem>(); BlobContinuationToken currentToken = null; IStorageBlobResultSegment result; do { result = await client.ListBlobsSegmentedAsync(prefix, useFlatBlobListing, blobListingDetails, maxResults : null, currentToken : currentToken, options : null, operationContext : null, cancellationToken : cancellationToken); if (result != null) { IEnumerable <IStorageListBlobItem> currentResults = result.Results; if (currentResults != null) { allResults.AddRange(currentResults); } } }while (result != null && result.ContinuationToken != null); return(allResults); }
private IListener CreateQueueMessageToTriggerExecutionListener( ISharedContextProvider sharedContextProvider, SharedQueueWatcher sharedQueueWatcher, IStorageQueueClient queueClient, IStorageQueue hostBlobTriggerQueue, IStorageBlobClient blobClient, IBlobWrittenWatcher blobWrittenWatcher) { SharedBlobQueueListener sharedListener = sharedContextProvider.GetOrCreate<SharedBlobQueueListener>( new SharedBlobQueueListenerFactory(sharedQueueWatcher, queueClient, hostBlobTriggerQueue, blobClient, _queueConfiguration, _backgroundExceptionDispatcher, _log, blobWrittenWatcher)); sharedListener.Register(_functionId, _executor); return new BlobListener(sharedListener); }
public BlobReceiptManager(IStorageBlobClient client) : this(client.GetContainerReference(HostContainerNames.Hosts) .GetDirectoryReference(HostDirectoryNames.BlobReceipts)) { }
private void RegisterWithSharedBlobQueueListenerAsync( SharedBlobQueueListener sharedBlobQueueListener, IStorageBlobClient blobClient) { BlobQueueRegistration registration = new BlobQueueRegistration { Executor = _executor, BlobClient = blobClient }; sharedBlobQueueListener.Register(_functionId, registration); }
private static IStorageBlobContainer GetContainerReference(IStorageAccount account, string containerName) { IStorageBlobClient client = account.CreateBlobClient(); return(client.GetContainerReference(ContainerName)); }
private BlobLogListener(IStorageBlobClient blobClient) { _blobClient = blobClient; }
private async Task<IListener> CreateBlobDiscoveryToQueueMessageListenerAsync(string hostId, SharedBlobListener sharedBlobListener, IStorageBlobClient blobClient, IStorageQueue hostBlobTriggerQueue, IMessageEnqueuedWatcher messageEnqueuedWatcher, CancellationToken cancellationToken) { BlobTriggerExecutor triggerExecutor = new BlobTriggerExecutor(hostId, _functionId, _input, BlobETagReader.Instance, new BlobReceiptManager(blobClient), new BlobTriggerQueueWriter(hostBlobTriggerQueue, messageEnqueuedWatcher)); await sharedBlobListener.RegisterAsync(_container, triggerExecutor, cancellationToken); return new BlobListener(sharedBlobListener); }
public SharedBlobQueueListenerFactory(IFunctionExecutor executor, SharedQueueWatcher sharedQueueWatcher, IStorageQueueClient queueClient, IStorageQueue hostBlobTriggerQueue, IStorageBlobClient blobClient, IQueueConfiguration queueConfiguration, IBackgroundExceptionDispatcher backgroundExceptionDispatcher, TextWriter log, IBlobWrittenWatcher blobWrittenWatcher) { if (executor == null) { throw new ArgumentNullException("executor"); } if (sharedQueueWatcher == null) { throw new ArgumentNullException("sharedQueueWatcher"); } if (queueClient == null) { throw new ArgumentNullException("queueClient"); } if (hostBlobTriggerQueue == null) { throw new ArgumentNullException("hostBlobTriggerQueue"); } if (blobClient == null) { throw new ArgumentNullException("blobClient"); } if (queueConfiguration == null) { throw new ArgumentNullException("queueConfiguration"); } if (backgroundExceptionDispatcher == null) { throw new ArgumentNullException("backgroundExceptionDispatcher"); } if (log == null) { throw new ArgumentNullException("log"); } if (blobWrittenWatcher == null) { throw new ArgumentNullException("blobWrittenWatcher"); } _executor = executor; _sharedQueueWatcher = sharedQueueWatcher; _queueClient = queueClient; _hostBlobTriggerQueue = hostBlobTriggerQueue; _blobClient = blobClient; _queueConfiguration = queueConfiguration; _backgroundExceptionDispatcher = backgroundExceptionDispatcher; _log = log; _blobWrittenWatcher = blobWrittenWatcher; }
public BlobQueueTriggerExecutor(IStorageBlobClient client, IBlobWrittenWatcher blobWrittenWatcher) : this(client, BlobETagReader.Instance, BlobCausalityReader.Instance, blobWrittenWatcher) { }
/// <summary>Initializes a new instance of the <see cref="PersistentQueueWriter{T}"/> class.</summary> /// <param name="client"> /// A blob client for the storage account into which host output messages are written. /// </param> public PersistentQueueWriter(IStorageBlobClient client) : this(client.GetContainerReference(ContainerNames.HostOutput)) { }
public BlobTriggerBinding(ParameterInfo parameter, IArgumentBinding <IStorageBlob> argumentBinding, IStorageAccount hostAccount, IStorageAccount dataAccount, IBlobPathSource path, IHostIdProvider hostIdProvider, IQueueConfiguration queueConfiguration, IBackgroundExceptionDispatcher backgroundExceptionDispatcher, IContextSetter <IBlobWrittenWatcher> blobWrittenWatcherSetter, IContextSetter <IMessageEnqueuedWatcher> messageEnqueuedWatcherSetter, ISharedContextProvider sharedContextProvider, SingletonManager singletonManager, TraceWriter trace) { if (parameter == null) { throw new ArgumentNullException("parameter"); } if (argumentBinding == null) { throw new ArgumentNullException("argumentBinding"); } if (hostAccount == null) { throw new ArgumentNullException("hostAccount"); } if (dataAccount == null) { throw new ArgumentNullException("dataAccount"); } if (path == null) { throw new ArgumentNullException("path"); } if (hostIdProvider == null) { throw new ArgumentNullException("hostIdProvider"); } if (queueConfiguration == null) { throw new ArgumentNullException("queueConfiguration"); } if (backgroundExceptionDispatcher == null) { throw new ArgumentNullException("backgroundExceptionDispatcher"); } if (blobWrittenWatcherSetter == null) { throw new ArgumentNullException("blobWrittenWatcherSetter"); } if (messageEnqueuedWatcherSetter == null) { throw new ArgumentNullException("messageEnqueuedWatcherSetter"); } if (sharedContextProvider == null) { throw new ArgumentNullException("sharedContextProvider"); } if (singletonManager == null) { throw new ArgumentNullException("singletonManager"); } if (trace == null) { throw new ArgumentNullException("trace"); } _parameter = parameter; _argumentBinding = argumentBinding; _hostAccount = hostAccount; _dataAccount = dataAccount; StorageClientFactoryContext context = new StorageClientFactoryContext { Parameter = parameter }; _blobClient = dataAccount.CreateBlobClient(context); _accountName = BlobClient.GetAccountName(_blobClient); _path = path; _hostIdProvider = hostIdProvider; _queueConfiguration = queueConfiguration; _backgroundExceptionDispatcher = backgroundExceptionDispatcher; _blobWrittenWatcherSetter = blobWrittenWatcherSetter; _messageEnqueuedWatcherSetter = messageEnqueuedWatcherSetter; _sharedContextProvider = sharedContextProvider; _singletonManager = singletonManager; _trace = trace; _converter = CreateConverter(_blobClient); _bindingDataContract = CreateBindingDataContract(path); }
private static IAsyncObjectToTypeConverter<IStorageBlob> CreateConverter(IStorageBlobClient client) { return new CompositeAsyncObjectToTypeConverter<IStorageBlob>( new OutputConverter<IStorageBlob>(new AsyncConverter<IStorageBlob, IStorageBlob>( new IdentityConverter<IStorageBlob>())), new OutputConverter<ICloudBlob>(new AsyncConverter<ICloudBlob, IStorageBlob>( new CloudBlobToStorageBlobConverter())), new OutputConverter<string>(new StringToStorageBlobConverter(client))); }
public BlobTriggerBinding(string parameterName, IArgumentBinding<IStorageBlob> argumentBinding, IStorageAccount account, IBlobPathSource path, IHostIdProvider hostIdProvider, IQueueConfiguration queueConfiguration, IBackgroundExceptionDispatcher backgroundExceptionDispatcher, IContextSetter<IBlobWrittenWatcher> blobWrittenWatcherSetter, IContextSetter<IMessageEnqueuedWatcher> messageEnqueuedWatcherSetter, ISharedContextProvider sharedContextProvider, TextWriter log) { if (argumentBinding == null) { throw new ArgumentNullException("argumentBinding"); } if (account == null) { throw new ArgumentNullException("account"); } if (path == null) { throw new ArgumentNullException("path"); } if (hostIdProvider == null) { throw new ArgumentNullException("hostIdProvider"); } if (queueConfiguration == null) { throw new ArgumentNullException("queueConfiguration"); } if (backgroundExceptionDispatcher == null) { throw new ArgumentNullException("backgroundExceptionDispatcher"); } if (blobWrittenWatcherSetter == null) { throw new ArgumentNullException("blobWrittenWatcherSetter"); } if (messageEnqueuedWatcherSetter == null) { throw new ArgumentNullException("messageEnqueuedWatcherSetter"); } if (sharedContextProvider == null) { throw new ArgumentNullException("sharedContextProvider"); } if (log == null) { throw new ArgumentNullException("log"); } _parameterName = parameterName; _argumentBinding = argumentBinding; _account = account; _client = account.CreateBlobClient(); _accountName = BlobClient.GetAccountName(_client); _path = path; _hostIdProvider = hostIdProvider; _queueConfiguration = queueConfiguration; _backgroundExceptionDispatcher = backgroundExceptionDispatcher; _blobWrittenWatcherSetter = blobWrittenWatcherSetter; _messageEnqueuedWatcherSetter = messageEnqueuedWatcherSetter; _sharedContextProvider = sharedContextProvider; _log = log; _converter = CreateConverter(_client); _bindingDataContract = CreateBindingDataContract(path); }