public Task <IStorageBlob> ConvertAsync(string input, CancellationToken cancellationToken) { BlobPath path = BlobPath.ParseAndValidate(input); IStorageBlobContainer container = _client.GetContainerReference(path.ContainerName); return(container.GetBlobReferenceFromServerAsync(path.BlobName, cancellationToken)); }
public async Task <IEnumerable <IStorageBlob> > GetRecentBlobWritesAsync(CancellationToken cancellationToken, int hoursWindow = DefaultScanHoursWindow) { List <IStorageBlob> blobs = new List <IStorageBlob>(); var time = DateTime.UtcNow; // will scan back 2 hours, which is enough to deal with clock sqew foreach (var blob in await ListRecentLogFilesAsync(_blobClient, time, cancellationToken, hoursWindow)) { bool isAdded = _scannedBlobNames.Add(blob.Name); if (!isAdded) { continue; } // Need to clear out cache. if (_scannedBlobNames.Count > 100 * 1000) { _scannedBlobNames.Clear(); } IEnumerable <StorageAnalyticsLogEntry> entries = await _parser.ParseLogAsync(blob, cancellationToken); IEnumerable <BlobPath> filteredBlobs = GetPathsForValidBlobWrites(entries); foreach (BlobPath path in filteredBlobs) { IStorageBlobContainer container = _blobClient.GetContainerReference(path.ContainerName); blobs.Add(container.GetBlockBlobReference(path.BlobName)); } } return(blobs); }
public async Task <IEnumerable <IStorageBlob> > GetRecentBlobWritesAsync(CancellationToken cancellationToken, int hoursWindow = DefaultScanHoursWindow) { List <IStorageBlob> blobs = new List <IStorageBlob>(); var time = DateTime.UtcNow; // will scan back 2 hours, which is enough to deal with clock sqew foreach (var blob in await ListRecentLogFilesAsync(_blobClient, time, cancellationToken, hoursWindow)) { bool isAdded = _scannedBlobNames.Add(blob.Name); if (!isAdded) { continue; } // Need to clear out cache. if (_scannedBlobNames.Count > 100 * 1000) { _scannedBlobNames.Clear(); } var parsedBlobPaths = from entry in await _parser.ParseLogAsync(blob, cancellationToken) where entry.IsBlobWrite select entry.ToBlobPath(); foreach (BlobPath path in parsedBlobPaths.Where(p => p != null)) { IStorageBlobContainer container = _blobClient.GetContainerReference(path.ContainerName); blobs.Add(container.GetBlockBlobReference(path.BlobName)); } } return(blobs); }
public async Task <IValueProvider> BindAsync(BindingContext context) { BlobPath boundPath = _path.Bind(context.BindingData); IStorageBlobContainer container = _client.GetContainerReference(boundPath.ContainerName); ValueBindingContext containerContext = new BlobValueBindingContext(boundPath, context.ValueContext); return(await BindBlobContainerAsync(container, containerContext)); }
public SingletonManager(IStorageBlobClient blobClient, IBackgroundExceptionDispatcher backgroundExceptionDispatcher, SingletonConfiguration config, TraceWriter trace) { _backgroundExceptionDispatcher = backgroundExceptionDispatcher; _directory = blobClient.GetContainerReference(HostContainerNames.Hosts) .GetDirectoryReference(HostDirectoryNames.SingletonLocks); _config = config; _trace = trace; }
private static IStorageBlob CreateBlobReference(string containerName, string blobName) { IStorageAccount account = CreateAccount(); IStorageBlobClient client = account.CreateBlobClient(); IStorageBlobContainer container = client.GetContainerReference(containerName); return(container.GetBlockBlobReference(blobName)); }
private static IStorageBlobContainer CreateContainer(IStorageAccount account, string containerName) { IStorageBlobClient client = account.CreateBlobClient(); IStorageBlobContainer container = client.GetContainerReference(containerName); container.CreateIfNotExists(); return(container); }
private IStorageBlockBlob GetBlockBlobReference(IStorageBlobClient blobClient, string hostId, string storageAccountName, string containerName) { string blobScanInfoDirectoryPath = string.Format(CultureInfo.InvariantCulture, "blobscaninfo/{0}", hostId); var blobScanInfoDirectory = blobClient.GetContainerReference(HostContainerNames.Hosts).GetDirectoryReference(blobScanInfoDirectoryPath); string blobName = string.Format(CultureInfo.InvariantCulture, "{0}/{1}/scanInfo", storageAccountName, containerName); return(blobScanInfoDirectory.GetBlockBlobReference(blobName)); }
public void ExecuteAsync_IfBlobDoesNotMatchPattern_ReturnsTrue() { // Arrange IStorageAccount account = CreateAccount(); IStorageBlobClient client = account.CreateBlobClient(); string containerName = "container"; IStorageBlobContainer container = client.GetContainerReference(containerName); IStorageBlobContainer otherContainer = client.GetContainerReference("other"); IBlobPathSource input = BlobPathSource.Create(containerName + "/{name}"); ITriggerExecutor <IStorageBlob> product = CreateProductUnderTest(input); IStorageBlob blob = otherContainer.GetBlockBlobReference("nonmatch"); // Act Task <bool> task = product.ExecuteAsync(blob, CancellationToken.None); // Assert Assert.True(task.Result); }
private async Task <IStorageBlobContainer> GetContainerAsync( BlobAttribute blobAttribute, CancellationToken cancellationToken) { IStorageBlobClient client = await GetClientAsync(blobAttribute, cancellationToken); BlobPath boundPath = BlobPath.ParseAndValidate(blobAttribute.BlobPath, isContainerBinding: true); IStorageBlobContainer container = client.GetContainerReference(boundPath.ContainerName); return(container); }
protected override IStorageBlobContainer GetContainer(string accountName) { // Get the container via a full connection string Task <IStorageAccount> task = _accountProvider.GetStorageAccountAsync(accountName, CancellationToken.None); IStorageAccount storageAccount = task.Result; // singleton requires block blobs, cannot be premium storageAccount.AssertTypeOneOf(StorageAccountType.GeneralPurpose, StorageAccountType.BlobOnly); IStorageBlobClient blobClient = storageAccount.CreateBlobClient(); var container = blobClient.GetContainerReference(HostContainerNames.Hosts); return(container); }
// Write-only rule. async Task <CloudBlobDirectory> IAsyncConverter <BlobAttribute, CloudBlobDirectory> .ConvertAsync( BlobAttribute blobAttribute, CancellationToken cancellationToken) { IStorageBlobClient client = await GetClientAsync(blobAttribute, cancellationToken); BlobPath boundPath = BlobPath.ParseAndValidate(blobAttribute.BlobPath, isContainerBinding: false); IStorageBlobContainer container = client.GetContainerReference(boundPath.ContainerName); CloudBlobDirectory directory = container.SdkObject.GetDirectoryReference( boundPath.BlobName); return(directory); }
public IFunctionDefinition CreateFunctionDefinition(IReadOnlyDictionary <string, IBinding> nonTriggerBindings, IFunctionInvoker invoker, FunctionDescriptor functionDescriptor) { ITriggeredFunctionBinding <IStorageBlob> functionBinding = new TriggeredFunctionBinding <IStorageBlob>(_parameterName, this, nonTriggerBindings); ITriggeredFunctionInstanceFactory <IStorageBlob> instanceFactory = new TriggeredFunctionInstanceFactory <IStorageBlob>(functionBinding, invoker, functionDescriptor); IStorageBlobContainer container = _client.GetContainerReference(_path.ContainerNamePattern); IListenerFactory listenerFactory = new BlobListenerFactory(_hostIdProvider, _queueConfiguration, _backgroundExceptionDispatcher, _blobWrittenWatcherSetter, _messageEnqueuedWatcherSetter, _sharedContextProvider, _log, functionDescriptor.Id, _account, container, _path, instanceFactory); return(new FunctionDefinition(instanceFactory, listenerFactory)); }
public FunctionStatusLogger(IHostIdProvider hostIdProvider, IStorageBlobClient blobClient) { if (hostIdProvider == null) { throw new ArgumentNullException("hostIdProvider"); } if (blobClient == null) { throw new ArgumentNullException("blobClient"); } _hostIdProvider = hostIdProvider; _blobClient = blobClient; _hostContainer = _blobClient.GetContainerReference(HostContainerNames.Hosts); }
public Task <IListener> CreateListenerAsync(ListenerFactoryContext context) { if (context == null) { throw new ArgumentNullException("context"); } IStorageBlobContainer container = _blobClient.GetContainerReference(_path.ContainerNamePattern); var factory = new BlobListenerFactory(_hostIdProvider, _queueConfiguration, _backgroundExceptionDispatcher, _blobWrittenWatcherSetter, _messageEnqueuedWatcherSetter, _sharedContextProvider, _trace, context.Descriptor.Id, _hostAccount, _dataAccount, container, _path, context.Executor, _singletonManager); return(factory.CreateAsync(context.CancellationToken)); }
public async Task <IValueProvider> BindAsync(BindingContext context) { BlobPath boundPath = _path.Bind(context.BindingData); IStorageBlobContainer container = _client.GetContainerReference(boundPath.ContainerName); if (_argumentBinding.Access != FileAccess.Read) { await container.CreateIfNotExistsAsync(context.CancellationToken); } Type argumentType = _argumentBinding.ValueType; string blobName = boundPath.BlobName; IStorageBlob blob = await container.GetBlobReferenceForArgumentTypeAsync(blobName, argumentType, context.CancellationToken); return(await BindBlobAsync(blob, context.ValueContext)); }
internal static bool TryConvert(object value, IStorageBlobClient client, out IStorageBlobContainer container, out BlobPath path) { container = null; path = null; string fullPath = value as string; if (fullPath != null) { path = BlobPath.ParseAndValidate(fullPath, isContainerBinding: true); container = client.GetContainerReference(path.ContainerName); return(true); } return(false); }
public async Task <IStorageBlob> ConvertAsync(string input, CancellationToken cancellationToken) { BlobPath path; // For convenience, treat an an empty string as a request for the default value. if (String.IsNullOrEmpty(input) && _defaultPath.IsBound) { path = _defaultPath.Bind(null); } else { path = BlobPath.ParseAndValidate(input); } IStorageBlobContainer container = _client.GetContainerReference(path.ContainerName); await container.CreateIfNotExistsAsync(cancellationToken); return(await container.GetBlobReferenceForArgumentTypeAsync(path.BlobName, _argumentType, cancellationToken)); }
internal IStorageBlobDirectory GetLockDirectory(string accountName) { if (string.IsNullOrEmpty(accountName)) { accountName = ConnectionStringNames.Storage; } IStorageBlobDirectory storageDirectory = null; if (!_lockDirectoryMap.TryGetValue(accountName, out storageDirectory)) { Task <IStorageAccount> task = _accountProvider.GetAccountAsync(accountName, CancellationToken.None); IStorageAccount storageAccount = task.Result; IStorageBlobClient blobClient = storageAccount.CreateBlobClient(); storageDirectory = blobClient.GetContainerReference(HostContainerNames.Hosts) .GetDirectoryReference(HostDirectoryNames.SingletonLocks); _lockDirectoryMap[accountName] = storageDirectory; } return(storageDirectory); }
private async Task <IStorageBlob> GetBlobAsync( BlobAttribute blobAttribute, CancellationToken cancellationToken, Type requestedType = null) { IStorageBlobClient client = await GetClientAsync(blobAttribute, cancellationToken); BlobPath boundPath = BlobPath.ParseAndValidate(blobAttribute.BlobPath); IStorageBlobContainer container = client.GetContainerReference(boundPath.ContainerName); if (blobAttribute.Access != FileAccess.Read) { await container.CreateIfNotExistsAsync(cancellationToken); } IStorageBlob blob = await container.GetBlobReferenceForArgumentTypeAsync( boundPath.BlobName, requestedType, cancellationToken); return(blob); }
public StorageBlobScanInfoManager(string hostId, IStorageBlobClient blobClient) { if (string.IsNullOrEmpty(hostId)) { throw new ArgumentNullException("hostId"); } if (blobClient == null) { throw new ArgumentNullException("blobClient"); } _hostId = hostId; JsonSerializerSettings settings = new JsonSerializerSettings { DateFormatHandling = DateFormatHandling.IsoDateFormat }; _serializer = JsonSerializer.Create(settings); string blobScanInfoDirectoryPath = string.Format(CultureInfo.InvariantCulture, "blobscaninfo/{0}", _hostId); _blobScanInfoDirectory = blobClient.GetContainerReference(HostContainerNames.Hosts).GetDirectoryReference(blobScanInfoDirectoryPath); }
internal IStorageBlobDirectory GetLockDirectory(string accountName) { if (string.IsNullOrEmpty(accountName)) { accountName = ConnectionStringNames.Storage; } IStorageBlobDirectory storageDirectory = null; if (!_lockDirectoryMap.TryGetValue(accountName, out storageDirectory)) { Task <IStorageAccount> task = _accountProvider.GetStorageAccountAsync(accountName, CancellationToken.None); IStorageAccount storageAccount = task.Result; // singleton requires block blobs, cannot be premium storageAccount.AssertTypeOneOf(StorageAccountType.GeneralPurpose, StorageAccountType.BlobOnly); IStorageBlobClient blobClient = storageAccount.CreateBlobClient(); storageDirectory = blobClient.GetContainerReference(HostContainerNames.Hosts) .GetDirectoryReference(HostDirectoryNames.SingletonLocks); _lockDirectoryMap[accountName] = storageDirectory; } return(storageDirectory); }
/// <summary>Initializes a new instance of the <see cref="PersistentQueueWriter{T}"/> class.</summary> /// <param name="client"> /// A blob client for the storage account into which host output messages are written. /// </param> public PersistentQueueWriter(IStorageBlobClient client) : this(client.GetContainerReference(ContainerNames.HostOutput)) { }
internal static bool TryConvert(object value, IStorageBlobClient client, out IStorageBlobContainer container, out BlobPath path) { container = null; path = null; string fullPath = value as string; if (fullPath != null) { path = BlobPath.ParseAndValidate(fullPath, isContainerBinding: true); container = client.GetContainerReference(path.ContainerName); return true; } return false; }
public async Task <FunctionResult> ExecuteAsync(IStorageQueueMessage value, CancellationToken cancellationToken) { BlobTriggerMessage message = JsonConvert.DeserializeObject <BlobTriggerMessage>(value.AsString, JsonSerialization.Settings); if (message == null) { throw new InvalidOperationException("Invalid blob trigger message."); } string functionId = message.FunctionId; if (functionId == null) { throw new InvalidOperationException("Invalid function ID."); } // Ensure that the function ID is still valid. Otherwise, ignore this message. FunctionResult successResult = new FunctionResult(true); ITriggeredFunctionExecutor executor; if (!_registrations.TryGetValue(functionId, out executor)) { return(successResult); } IStorageBlobContainer container = _client.GetContainerReference(message.ContainerName); string blobName = message.BlobName; IStorageBlob blob; switch (message.BlobType) { case StorageBlobType.PageBlob: blob = container.GetPageBlobReference(blobName); break; case StorageBlobType.BlockBlob: default: blob = container.GetBlockBlobReference(blobName); break; } // Ensure the blob still exists with the same ETag. string possibleETag = await _eTagReader.GetETagAsync(blob, cancellationToken); if (possibleETag == null) { // If the blob no longer exists, just ignore this message. return(successResult); } // If the blob still exists but the ETag is different, delete the message but do a fast path notification. if (!String.Equals(message.ETag, possibleETag, StringComparison.Ordinal)) { _blobWrittenWatcher.Notify(blob); return(successResult); } //// If the blob still exists and its ETag is still valid, execute. //// Note: it's possible the blob could change/be deleted between now and when the function executes. Guid?parentId = await _causalityReader.GetWriterAsync(blob, cancellationToken); TriggeredFunctionData input = new TriggeredFunctionData { ParentId = parentId, TriggerValue = blob }; return(await executor.TryExecuteAsync(input, cancellationToken)); }
public BlobFunctionOutputLogger(IStorageBlobClient client) : this(client.GetContainerReference(HostContainerNames.Hosts).GetDirectoryReference(HostDirectoryNames.OutputLogs)) { }
public BlobReceiptManager(IStorageBlobClient client) : this(client.GetContainerReference(HostContainerNames.Hosts) .GetDirectoryReference(HostDirectoryNames.BlobReceipts)) { }
public BlobFunctionOutputLogger(IStorageBlobClient client) : this(client.GetContainerReference( HostContainerNames.Hosts).GetDirectoryReference(HostDirectoryNames.OutputLogs)) { }
private static IStorageBlobContainer GetContainerReference(IStorageAccount account, string containerName) { IStorageBlobClient client = account.CreateBlobClient(); return(client.GetContainerReference(ContainerName)); }
private IStorageBlockBlob GetBlockBlobReference(LocalBlobDescriptor descriptor) { IStorageBlobContainer container = _client.GetContainerReference(descriptor.ContainerName); return(container.GetBlockBlobReference(descriptor.BlobName)); }