public Task <IStorageBlob> ConvertAsync(string input, CancellationToken cancellationToken)
{
BlobPath path = BlobPath.ParseAndValidate(input);
IStorageBlobContainer container = _client.GetContainerReference(path.ContainerName);
return(container.GetBlobReferenceFromServerAsync(path.BlobName, cancellationToken));
}
public async Task <IEnumerable <IStorageBlob> > GetRecentBlobWritesAsync(CancellationToken cancellationToken,
int hoursWindow = DefaultScanHoursWindow)
{
List <IStorageBlob> blobs = new List <IStorageBlob>();
var time = DateTime.UtcNow; // will scan back 2 hours, which is enough to deal with clock sqew
foreach (var blob in await ListRecentLogFilesAsync(_blobClient, time, cancellationToken, hoursWindow))
{
bool isAdded = _scannedBlobNames.Add(blob.Name);
if (!isAdded)
{
continue;
}
// Need to clear out cache.
if (_scannedBlobNames.Count > 100 * 1000)
{
_scannedBlobNames.Clear();
}
IEnumerable <StorageAnalyticsLogEntry> entries = await _parser.ParseLogAsync(blob, cancellationToken);
IEnumerable <BlobPath> filteredBlobs = GetPathsForValidBlobWrites(entries);
foreach (BlobPath path in filteredBlobs)
{
IStorageBlobContainer container = _blobClient.GetContainerReference(path.ContainerName);
blobs.Add(container.GetBlockBlobReference(path.BlobName));
}
}
return(blobs);
}
public async Task <IEnumerable <IStorageBlob> > GetRecentBlobWritesAsync(CancellationToken cancellationToken,
int hoursWindow = DefaultScanHoursWindow)
{
List <IStorageBlob> blobs = new List <IStorageBlob>();
var time = DateTime.UtcNow; // will scan back 2 hours, which is enough to deal with clock sqew
foreach (var blob in await ListRecentLogFilesAsync(_blobClient, time, cancellationToken, hoursWindow))
{
bool isAdded = _scannedBlobNames.Add(blob.Name);
if (!isAdded)
{
continue;
}
// Need to clear out cache.
if (_scannedBlobNames.Count > 100 * 1000)
{
_scannedBlobNames.Clear();
}
var parsedBlobPaths = from entry in await _parser.ParseLogAsync(blob, cancellationToken)
where entry.IsBlobWrite
select entry.ToBlobPath();
foreach (BlobPath path in parsedBlobPaths.Where(p => p != null))
{
IStorageBlobContainer container = _blobClient.GetContainerReference(path.ContainerName);
blobs.Add(container.GetBlockBlobReference(path.BlobName));
}
}
return(blobs);
}
public async Task <IValueProvider> BindAsync(BindingContext context)
{
BlobPath boundPath = _path.Bind(context.BindingData);
IStorageBlobContainer container = _client.GetContainerReference(boundPath.ContainerName);
ValueBindingContext containerContext = new BlobValueBindingContext(boundPath, context.ValueContext);
return(await BindBlobContainerAsync(container, containerContext));
}
public SingletonManager(IStorageBlobClient blobClient, IBackgroundExceptionDispatcher backgroundExceptionDispatcher, SingletonConfiguration config, TraceWriter trace)
{
_backgroundExceptionDispatcher = backgroundExceptionDispatcher;
_directory = blobClient.GetContainerReference(HostContainerNames.Hosts)
.GetDirectoryReference(HostDirectoryNames.SingletonLocks);
_config = config;
_trace = trace;
}
public SingletonManager(IStorageBlobClient blobClient, IBackgroundExceptionDispatcher backgroundExceptionDispatcher, SingletonConfiguration config, TraceWriter trace)
{
_backgroundExceptionDispatcher = backgroundExceptionDispatcher;
_directory = blobClient.GetContainerReference(HostContainerNames.Hosts)
.GetDirectoryReference(HostDirectoryNames.SingletonLocks);
_config = config;
_trace = trace;
}
private static IStorageBlob CreateBlobReference(string containerName, string blobName)
{
IStorageAccount account = CreateAccount();
IStorageBlobClient client = account.CreateBlobClient();
IStorageBlobContainer container = client.GetContainerReference(containerName);
return(container.GetBlockBlobReference(blobName));
}
private static IStorageBlobContainer CreateContainer(IStorageAccount account, string containerName)
{
IStorageBlobClient client = account.CreateBlobClient();
IStorageBlobContainer container = client.GetContainerReference(containerName);
container.CreateIfNotExists();
return(container);
}
private IStorageBlockBlob GetBlockBlobReference(IStorageBlobClient blobClient, string hostId, string storageAccountName, string containerName)
{
string blobScanInfoDirectoryPath = string.Format(CultureInfo.InvariantCulture, "blobscaninfo/{0}", hostId);
var blobScanInfoDirectory = blobClient.GetContainerReference(HostContainerNames.Hosts).GetDirectoryReference(blobScanInfoDirectoryPath);
string blobName = string.Format(CultureInfo.InvariantCulture, "{0}/{1}/scanInfo", storageAccountName, containerName);
return(blobScanInfoDirectory.GetBlockBlobReference(blobName));
}
public void ExecuteAsync_IfBlobDoesNotMatchPattern_ReturnsTrue()
{
// Arrange
IStorageAccount account = CreateAccount();
IStorageBlobClient client = account.CreateBlobClient();
string containerName = "container";
IStorageBlobContainer container = client.GetContainerReference(containerName);
IStorageBlobContainer otherContainer = client.GetContainerReference("other");
IBlobPathSource input = BlobPathSource.Create(containerName + "/{name}");
ITriggerExecutor <IStorageBlob> product = CreateProductUnderTest(input);
IStorageBlob blob = otherContainer.GetBlockBlobReference("nonmatch");
// Act
Task <bool> task = product.ExecuteAsync(blob, CancellationToken.None);
// Assert
Assert.True(task.Result);
}
private async Task <IStorageBlobContainer> GetContainerAsync(
BlobAttribute blobAttribute,
CancellationToken cancellationToken)
{
IStorageBlobClient client = await GetClientAsync(blobAttribute, cancellationToken);
BlobPath boundPath = BlobPath.ParseAndValidate(blobAttribute.BlobPath, isContainerBinding: true);
IStorageBlobContainer container = client.GetContainerReference(boundPath.ContainerName);
return(container);
}
protected override IStorageBlobContainer GetContainer(string accountName)
{
// Get the container via a full connection string
Task <IStorageAccount> task = _accountProvider.GetStorageAccountAsync(accountName, CancellationToken.None);
IStorageAccount storageAccount = task.Result;
// singleton requires block blobs, cannot be premium
storageAccount.AssertTypeOneOf(StorageAccountType.GeneralPurpose, StorageAccountType.BlobOnly);
IStorageBlobClient blobClient = storageAccount.CreateBlobClient();
var container = blobClient.GetContainerReference(HostContainerNames.Hosts);
return(container);
}
// Write-only rule.
async Task <CloudBlobDirectory> IAsyncConverter <BlobAttribute, CloudBlobDirectory> .ConvertAsync(
BlobAttribute blobAttribute, CancellationToken cancellationToken)
{
IStorageBlobClient client = await GetClientAsync(blobAttribute, cancellationToken);
BlobPath boundPath = BlobPath.ParseAndValidate(blobAttribute.BlobPath, isContainerBinding: false);
IStorageBlobContainer container = client.GetContainerReference(boundPath.ContainerName);
CloudBlobDirectory directory = container.SdkObject.GetDirectoryReference(
boundPath.BlobName);
return(directory);
}
public IFunctionDefinition CreateFunctionDefinition(IReadOnlyDictionary <string, IBinding> nonTriggerBindings,
IFunctionInvoker invoker, FunctionDescriptor functionDescriptor)
{
ITriggeredFunctionBinding <IStorageBlob> functionBinding =
new TriggeredFunctionBinding <IStorageBlob>(_parameterName, this, nonTriggerBindings);
ITriggeredFunctionInstanceFactory <IStorageBlob> instanceFactory =
new TriggeredFunctionInstanceFactory <IStorageBlob>(functionBinding, invoker, functionDescriptor);
IStorageBlobContainer container = _client.GetContainerReference(_path.ContainerNamePattern);
IListenerFactory listenerFactory = new BlobListenerFactory(_hostIdProvider, _queueConfiguration,
_backgroundExceptionDispatcher, _blobWrittenWatcherSetter, _messageEnqueuedWatcherSetter,
_sharedContextProvider, _log, functionDescriptor.Id, _account, container, _path, instanceFactory);
return(new FunctionDefinition(instanceFactory, listenerFactory));
}
public FunctionStatusLogger(IHostIdProvider hostIdProvider, IStorageBlobClient blobClient)
{
if (hostIdProvider == null)
{
throw new ArgumentNullException("hostIdProvider");
}
if (blobClient == null)
{
throw new ArgumentNullException("blobClient");
}
_hostIdProvider = hostIdProvider;
_blobClient = blobClient;
_hostContainer = _blobClient.GetContainerReference(HostContainerNames.Hosts);
}
public Task <IListener> CreateListenerAsync(ListenerFactoryContext context)
{
if (context == null)
{
throw new ArgumentNullException("context");
}
IStorageBlobContainer container = _blobClient.GetContainerReference(_path.ContainerNamePattern);
var factory = new BlobListenerFactory(_hostIdProvider, _queueConfiguration,
_backgroundExceptionDispatcher, _blobWrittenWatcherSetter, _messageEnqueuedWatcherSetter,
_sharedContextProvider, _trace, context.Descriptor.Id, _hostAccount, _dataAccount, container, _path, context.Executor, _singletonManager);
return(factory.CreateAsync(context.CancellationToken));
}
public async Task <IValueProvider> BindAsync(BindingContext context)
{
BlobPath boundPath = _path.Bind(context.BindingData);
IStorageBlobContainer container = _client.GetContainerReference(boundPath.ContainerName);
if (_argumentBinding.Access != FileAccess.Read)
{
await container.CreateIfNotExistsAsync(context.CancellationToken);
}
Type argumentType = _argumentBinding.ValueType;
string blobName = boundPath.BlobName;
IStorageBlob blob = await container.GetBlobReferenceForArgumentTypeAsync(blobName, argumentType, context.CancellationToken);
return(await BindBlobAsync(blob, context.ValueContext));
}
internal static bool TryConvert(object value, IStorageBlobClient client, out IStorageBlobContainer container, out BlobPath path)
{
container = null;
path = null;
string fullPath = value as string;
if (fullPath != null)
{
path = BlobPath.ParseAndValidate(fullPath, isContainerBinding: true);
container = client.GetContainerReference(path.ContainerName);
return(true);
}
return(false);
}
public async Task <IStorageBlob> ConvertAsync(string input, CancellationToken cancellationToken)
{
BlobPath path;
// For convenience, treat an an empty string as a request for the default value.
if (String.IsNullOrEmpty(input) && _defaultPath.IsBound)
{
path = _defaultPath.Bind(null);
}
else
{
path = BlobPath.ParseAndValidate(input);
}
IStorageBlobContainer container = _client.GetContainerReference(path.ContainerName);
await container.CreateIfNotExistsAsync(cancellationToken);
return(await container.GetBlobReferenceForArgumentTypeAsync(path.BlobName, _argumentType,
cancellationToken));
}
internal IStorageBlobDirectory GetLockDirectory(string accountName)
{
if (string.IsNullOrEmpty(accountName))
{
accountName = ConnectionStringNames.Storage;
}
IStorageBlobDirectory storageDirectory = null;
if (!_lockDirectoryMap.TryGetValue(accountName, out storageDirectory))
{
Task <IStorageAccount> task = _accountProvider.GetAccountAsync(accountName, CancellationToken.None);
IStorageAccount storageAccount = task.Result;
IStorageBlobClient blobClient = storageAccount.CreateBlobClient();
storageDirectory = blobClient.GetContainerReference(HostContainerNames.Hosts)
.GetDirectoryReference(HostDirectoryNames.SingletonLocks);
_lockDirectoryMap[accountName] = storageDirectory;
}
return(storageDirectory);
}
private async Task <IStorageBlob> GetBlobAsync(
BlobAttribute blobAttribute,
CancellationToken cancellationToken,
Type requestedType = null)
{
IStorageBlobClient client = await GetClientAsync(blobAttribute, cancellationToken);
BlobPath boundPath = BlobPath.ParseAndValidate(blobAttribute.BlobPath);
IStorageBlobContainer container = client.GetContainerReference(boundPath.ContainerName);
if (blobAttribute.Access != FileAccess.Read)
{
await container.CreateIfNotExistsAsync(cancellationToken);
}
IStorageBlob blob = await container.GetBlobReferenceForArgumentTypeAsync(
boundPath.BlobName, requestedType, cancellationToken);
return(blob);
}
public StorageBlobScanInfoManager(string hostId, IStorageBlobClient blobClient)
{
if (string.IsNullOrEmpty(hostId))
{
throw new ArgumentNullException("hostId");
}
if (blobClient == null)
{
throw new ArgumentNullException("blobClient");
}
_hostId = hostId;
JsonSerializerSettings settings = new JsonSerializerSettings
{
DateFormatHandling = DateFormatHandling.IsoDateFormat
};
_serializer = JsonSerializer.Create(settings);
string blobScanInfoDirectoryPath = string.Format(CultureInfo.InvariantCulture, "blobscaninfo/{0}", _hostId);
_blobScanInfoDirectory = blobClient.GetContainerReference(HostContainerNames.Hosts).GetDirectoryReference(blobScanInfoDirectoryPath);
}
internal IStorageBlobDirectory GetLockDirectory(string accountName)
{
if (string.IsNullOrEmpty(accountName))
{
accountName = ConnectionStringNames.Storage;
}
IStorageBlobDirectory storageDirectory = null;
if (!_lockDirectoryMap.TryGetValue(accountName, out storageDirectory))
{
Task <IStorageAccount> task = _accountProvider.GetStorageAccountAsync(accountName, CancellationToken.None);
IStorageAccount storageAccount = task.Result;
// singleton requires block blobs, cannot be premium
storageAccount.AssertTypeOneOf(StorageAccountType.GeneralPurpose, StorageAccountType.BlobOnly);
IStorageBlobClient blobClient = storageAccount.CreateBlobClient();
storageDirectory = blobClient.GetContainerReference(HostContainerNames.Hosts)
.GetDirectoryReference(HostDirectoryNames.SingletonLocks);
_lockDirectoryMap[accountName] = storageDirectory;
}
return(storageDirectory);
}
/// <summary>Initializes a new instance of the <see cref="PersistentQueueWriter{T}"/> class.</summary>
/// <param name="client">
/// A blob client for the storage account into which host output messages are written.
/// </param>
public PersistentQueueWriter(IStorageBlobClient client)
: this(client.GetContainerReference(ContainerNames.HostOutput))
{
}
internal static bool TryConvert(object value, IStorageBlobClient client, out IStorageBlobContainer container, out BlobPath path)
{
container = null;
path = null;
string fullPath = value as string;
if (fullPath != null)
{
path = BlobPath.ParseAndValidate(fullPath, isContainerBinding: true);
container = client.GetContainerReference(path.ContainerName);
return true;
}
return false;
}
public async Task <FunctionResult> ExecuteAsync(IStorageQueueMessage value, CancellationToken cancellationToken)
{
BlobTriggerMessage message = JsonConvert.DeserializeObject <BlobTriggerMessage>(value.AsString, JsonSerialization.Settings);
if (message == null)
{
throw new InvalidOperationException("Invalid blob trigger message.");
}
string functionId = message.FunctionId;
if (functionId == null)
{
throw new InvalidOperationException("Invalid function ID.");
}
// Ensure that the function ID is still valid. Otherwise, ignore this message.
FunctionResult successResult = new FunctionResult(true);
ITriggeredFunctionExecutor executor;
if (!_registrations.TryGetValue(functionId, out executor))
{
return(successResult);
}
IStorageBlobContainer container = _client.GetContainerReference(message.ContainerName);
string blobName = message.BlobName;
IStorageBlob blob;
switch (message.BlobType)
{
case StorageBlobType.PageBlob:
blob = container.GetPageBlobReference(blobName);
break;
case StorageBlobType.BlockBlob:
default:
blob = container.GetBlockBlobReference(blobName);
break;
}
// Ensure the blob still exists with the same ETag.
string possibleETag = await _eTagReader.GetETagAsync(blob, cancellationToken);
if (possibleETag == null)
{
// If the blob no longer exists, just ignore this message.
return(successResult);
}
// If the blob still exists but the ETag is different, delete the message but do a fast path notification.
if (!String.Equals(message.ETag, possibleETag, StringComparison.Ordinal))
{
_blobWrittenWatcher.Notify(blob);
return(successResult);
}
//// If the blob still exists and its ETag is still valid, execute.
//// Note: it's possible the blob could change/be deleted between now and when the function executes.
Guid?parentId = await _causalityReader.GetWriterAsync(blob, cancellationToken);
TriggeredFunctionData input = new TriggeredFunctionData
{
ParentId = parentId,
TriggerValue = blob
};
return(await executor.TryExecuteAsync(input, cancellationToken));
}
public BlobFunctionOutputLogger(IStorageBlobClient client)
: this(client.GetContainerReference(HostContainerNames.Hosts).GetDirectoryReference(HostDirectoryNames.OutputLogs))
{
}
public BlobReceiptManager(IStorageBlobClient client)
: this(client.GetContainerReference(HostContainerNames.Hosts)
.GetDirectoryReference(HostDirectoryNames.BlobReceipts))
{
}
public BlobFunctionOutputLogger(IStorageBlobClient client)
: this(client.GetContainerReference(
HostContainerNames.Hosts).GetDirectoryReference(HostDirectoryNames.OutputLogs))
{
}
private static IStorageBlobContainer GetContainerReference(IStorageAccount account, string containerName)
{
IStorageBlobClient client = account.CreateBlobClient();
return(client.GetContainerReference(ContainerName));
}
private IStorageBlockBlob GetBlockBlobReference(LocalBlobDescriptor descriptor)
{
IStorageBlobContainer container = _client.GetContainerReference(descriptor.ContainerName);
return(container.GetBlockBlobReference(descriptor.BlobName));
}
public BlobReceiptManager(IStorageBlobClient client)
: this(client.GetContainerReference(HostContainerNames.Hosts)
.GetDirectoryReference(HostDirectoryNames.BlobReceipts))
{
}