/// <inheritdoc />
public async Task <string> EnqueueAsync(T message, CancellationToken cancellationToken)
{
await _blobContainer.CreateIfNotExistsAsync(cancellationToken);
string blobName = BlobNames.GetConflictFreeDateTimeBasedBlobName();
IStorageBlockBlob blob = _blobContainer.GetBlockBlobReference(blobName);
message.AddMetadata(blob.Metadata);
string messageBody = JsonConvert.SerializeObject(message, JsonSerialization.Settings);
await blob.UploadTextAsync(messageBody, cancellationToken : cancellationToken);
return(blobName);
}
public void BlobTrigger_IfBindingAlwaysFails_MovesToPoisonQueue()
{
// Arrange
IStorageAccount account = CreateFakeStorageAccount();
IStorageBlobContainer container = CreateContainer(account, ContainerName);
IStorageBlockBlob blob = container.GetBlockBlobReference(BlobName);
CloudBlockBlob expectedBlob = blob.SdkObject;
blob.UploadText("ignore");
// Act
string result = RunTrigger <string>(account, typeof(PoisonBlobProgram),
(s) => PoisonBlobProgram.TaskSource = s,
new string[] { typeof(PoisonBlobProgram).FullName + ".PutInPoisonQueue" });
// Assert
BlobTriggerMessage message = JsonConvert.DeserializeObject <BlobTriggerMessage>(result);
Assert.NotNull(message);
Assert.Equal(typeof(PoisonBlobProgram).FullName + ".PutInPoisonQueue", message.FunctionId);
Assert.Equal(StorageBlobType.BlockBlob, message.BlobType);
Assert.Equal(ContainerName, message.ContainerName);
Assert.Equal(BlobName, message.BlobName);
Assert.NotEmpty(message.ETag);
}
public async Task <IEnumerable <IStorageBlob> > GetRecentBlobWritesAsync(CancellationToken cancellationToken,
int hoursWindow = DefaultScanHoursWindow)
{
List <IStorageBlob> blobs = new List <IStorageBlob>();
var time = DateTime.UtcNow; // will scan back 2 hours, which is enough to deal with clock sqew
foreach (var blob in await ListRecentLogFilesAsync(_blobClient, time, cancellationToken, hoursWindow))
{
bool isAdded = _scannedBlobNames.Add(blob.Name);
if (!isAdded)
{
continue;
}
// Need to clear out cache.
if (_scannedBlobNames.Count > 100 * 1000)
{
_scannedBlobNames.Clear();
}
IEnumerable <StorageAnalyticsLogEntry> entries = await _parser.ParseLogAsync(blob, cancellationToken);
IEnumerable <BlobPath> filteredBlobs = GetPathsForValidBlobWrites(entries);
foreach (BlobPath path in filteredBlobs)
{
IStorageBlobContainer container = _blobClient.GetContainerReference(path.ContainerName);
blobs.Add(container.GetBlockBlobReference(path.BlobName));
}
}
return(blobs);
}
public async Task <IEnumerable <IStorageBlob> > GetRecentBlobWritesAsync(CancellationToken cancellationToken,
int hoursWindow = DefaultScanHoursWindow)
{
List <IStorageBlob> blobs = new List <IStorageBlob>();
var time = DateTime.UtcNow; // will scan back 2 hours, which is enough to deal with clock sqew
foreach (var blob in await ListRecentLogFilesAsync(_blobClient, time, cancellationToken, hoursWindow))
{
bool isAdded = _scannedBlobNames.Add(blob.Name);
if (!isAdded)
{
continue;
}
// Need to clear out cache.
if (_scannedBlobNames.Count > 100 * 1000)
{
_scannedBlobNames.Clear();
}
var parsedBlobPaths = from entry in await _parser.ParseLogAsync(blob, cancellationToken)
where entry.IsBlobWrite
select entry.ToBlobPath();
foreach (BlobPath path in parsedBlobPaths.Where(p => p != null))
{
IStorageBlobContainer container = _blobClient.GetContainerReference(path.ContainerName);
blobs.Add(container.GetBlockBlobReference(path.BlobName));
}
}
return(blobs);
}
private static IStorageBlob CreateBlobReference(string containerName, string blobName)
{
IStorageAccount account = CreateAccount();
IStorageBlobClient client = account.CreateBlobClient();
IStorageBlobContainer container = client.GetContainerReference(containerName);
return(container.GetBlockBlobReference(blobName));
}
private string CreateAblobAndUploadToContainer(IStorageBlobContainer container, string blobContent = "test")
{
string blobName = Path.GetRandomFileName().Replace(".", "");
IStorageBlockBlob blob = container.GetBlockBlobReference(blobName);
container.CreateIfNotExists();
blob.UploadText(blobContent);
return(blobName);
}
public void BlobTriggerToQueueTriggerToBlob_WritesFinalBlob()
{
// Arrange
IStorageAccount account = CreateFakeStorageAccount();
IStorageBlobContainer container = CreateContainer(account, ContainerName);
IStorageBlockBlob inputBlob = container.GetBlockBlobReference(BlobName);
inputBlob.UploadText("15");
// Act
RunTrigger <object>(account, typeof(BlobTriggerToQueueTriggerToBlobProgram),
(s) => BlobTriggerToQueueTriggerToBlobProgram.TaskSource = s);
// Assert
IStorageBlockBlob outputBlob = container.GetBlockBlobReference(OutputBlobName);
string content = outputBlob.DownloadText();
Assert.Equal("16", content);
}
public void BlobTrigger_IfWritesToSecondBlobTrigger_TriggersOutputQuickly()
{
// Arrange
IStorageAccount account = CreateFakeStorageAccount();
IStorageBlobContainer container = CreateContainer(account, ContainerName);
IStorageBlockBlob inputBlob = container.GetBlockBlobReference(BlobName);
inputBlob.UploadText("abc");
// Act
RunTrigger <object>(account, typeof(BlobTriggerToBlobTriggerProgram),
(s) => BlobTriggerToBlobTriggerProgram.TaskSource = s);
// Assert
IStorageBlockBlob outputBlob = container.GetBlockBlobReference(OutputBlobName);
string content = outputBlob.DownloadText();
Assert.Equal("*abc*", content);
}
public void BlobTrigger_Binding_Metadata()
{
var app = new BindToCloudBlob2Program();
var activator = new FakeActivator(app);
IStorageAccount account = CreateFakeStorageAccount();
var host = TestHelpers.NewJobHost <BindToCloudBlob2Program>(account, activator);
// Set the binding data, and verify it's accessible in the function.
IStorageBlobContainer container = CreateContainer(account, ContainerName);
IStorageBlockBlob blob = container.GetBlockBlobReference(BlobName);
blob.Metadata["m1"] = "v1";
host.Call("Run", new { blob = blob });
Assert.True(app.Success);
}
public void BlobTrigger_IfBoundToCloudBlob_Binds()
{
// Arrange
IStorageAccount account = CreateFakeStorageAccount();
IStorageBlobContainer container = CreateContainer(account, ContainerName);
IStorageBlockBlob blob = container.GetBlockBlobReference(BlobName);
CloudBlockBlob expectedBlob = blob.SdkObject;
blob.UploadText("ignore");
// Act
ICloudBlob result = RunTrigger <ICloudBlob>(account, typeof(BindToCloudBlobProgram),
(s) => BindToCloudBlobProgram.TaskSource = s);
// Assert
Assert.Equal(expectedBlob.Uri, result.Uri);
}
public static Task <IStorageBlob> GetBlobReferenceForArgumentTypeAsync(this IStorageBlobContainer container,
string blobName, Type argumentType, CancellationToken cancellationToken)
{
if (argumentType == typeof(CloudBlockBlob))
{
IStorageBlob blob = container.GetBlockBlobReference(blobName);
return(Task.FromResult(blob));
}
else if (argumentType == typeof(CloudPageBlob))
{
IStorageBlob blob = container.GetPageBlobReference(blobName);
return(Task.FromResult(blob));
}
else
{
return(GetExistingOrNewBlockBlobReferenceAsync(container, blobName, cancellationToken));
}
}
public void TestBlobListener()
{
const string containerName = "container";
IStorageAccount account = CreateFakeStorageAccount();
IStorageBlobContainer container = account.CreateBlobClient().GetContainerReference(containerName);
IBlobListenerStrategy product = new ScanContainersStrategy();
LambdaBlobTriggerExecutor executor = new LambdaBlobTriggerExecutor();
product.Register(container, executor);
product.Start();
executor.ExecuteLambda = (_) =>
{
throw new InvalidOperationException("shouldn't be any blobs in the container");
};
product.Execute();
const string expectedBlobName = "foo1.csv";
IStorageBlockBlob blob = container.GetBlockBlobReference(expectedBlobName);
container.CreateIfNotExists();
blob.UploadText("ignore");
int count = 0;
executor.ExecuteLambda = (b) =>
{
count++;
Assert.Equal(expectedBlobName, b.Name);
return(true);
};
product.Execute();
Assert.Equal(1, count);
// Now run again; shouldn't show up.
executor.ExecuteLambda = (_) =>
{
throw new InvalidOperationException("shouldn't retrigger the same blob");
};
product.Execute();
}
public void ExecuteAsync_IfBlobDoesNotMatchPattern_ReturnsTrue()
{
// Arrange
IStorageAccount account = CreateAccount();
IStorageBlobClient client = account.CreateBlobClient();
string containerName = "container";
IStorageBlobContainer container = client.GetContainerReference(containerName);
IStorageBlobContainer otherContainer = client.GetContainerReference("other");
IBlobPathSource input = BlobPathSource.Create(containerName + "/{name}");
ITriggerExecutor <IStorageBlob> product = CreateProductUnderTest(input);
IStorageBlob blob = otherContainer.GetBlockBlobReference("nonmatch");
// Act
Task <bool> task = product.ExecuteAsync(blob, CancellationToken.None);
// Assert
Assert.True(task.Result);
}
private static async Task <IStorageBlob> GetExistingOrNewBlockBlobReferenceAsync(IStorageBlobContainer container,
string blobName, CancellationToken cancellationToken)
{
try
{
return(await container.GetBlobReferenceFromServerAsync(blobName, cancellationToken));
}
catch (StorageException exception)
{
RequestResult result = exception.RequestInformation;
if (result == null || result.HttpStatusCode != 404)
{
throw;
}
else
{
return(container.GetBlockBlobReference(blobName));
}
}
}
private static async Task<IStorageBlob> GetExistingOrNewBlockBlobReferenceAsync(IStorageBlobContainer container,
string blobName, CancellationToken cancellationToken)
{
try
{
return await container.GetBlobReferenceFromServerAsync(blobName, cancellationToken);
}
catch (StorageException exception)
{
RequestResult result = exception.RequestInformation;
if (result == null || result.HttpStatusCode != 404)
{
throw;
}
else
{
return container.GetBlockBlobReference(blobName);
}
}
}
public void Blob_IfBoundToTextWriter_CreatesBlob()
{
// Arrange
const string expectedContent = "message";
IStorageAccount account = CreateFakeStorageAccount();
IStorageQueue triggerQueue = CreateQueue(account, TriggerQueueName);
triggerQueue.AddMessage(triggerQueue.CreateMessage(expectedContent));
// Act
RunTrigger(account, typeof(BindToTextWriterProgram));
// Assert
IStorageBlobContainer container = GetContainerReference(account, ContainerName);
Assert.True(container.Exists());
IStorageBlockBlob blob = container.GetBlockBlobReference(BlobName);
Assert.True(blob.Exists());
string content = blob.DownloadText();
Assert.Equal(expectedContent, content);
}
public void Blob_IfBoundToCloudBlockBlob_BindsAndCreatesContainerButNotBlob()
{
// Arrange
IStorageAccount account = CreateFakeStorageAccount();
IStorageQueue triggerQueue = CreateQueue(account, TriggerQueueName);
triggerQueue.AddMessage(triggerQueue.CreateMessage("ignore"));
// Act
CloudBlockBlob result = RunTrigger <CloudBlockBlob>(account, typeof(BindToCloudBlockBlobProgram),
(s) => BindToCloudBlockBlobProgram.TaskSource = s);
// Assert
Assert.NotNull(result);
Assert.Equal(BlobName, result.Name);
Assert.NotNull(result.Container);
Assert.Equal(ContainerName, result.Container.Name);
IStorageBlobContainer container = GetContainerReference(account, ContainerName);
Assert.True(container.Exists());
IStorageBlockBlob blob = container.GetBlockBlobReference(BlobName);
Assert.False(blob.Exists());
}
private IStorageBlockBlob GetBlockBlobReference(LocalBlobDescriptor descriptor)
{
IStorageBlobContainer container = _client.GetContainerReference(descriptor.ContainerName);
return(container.GetBlockBlobReference(descriptor.BlobName));
}
public async Task <FunctionResult> ExecuteAsync(IStorageQueueMessage value, CancellationToken cancellationToken)
{
BlobTriggerMessage message = JsonConvert.DeserializeObject <BlobTriggerMessage>(value.AsString, JsonSerialization.Settings);
if (message == null)
{
throw new InvalidOperationException("Invalid blob trigger message.");
}
string functionId = message.FunctionId;
if (functionId == null)
{
throw new InvalidOperationException("Invalid function ID.");
}
// Ensure that the function ID is still valid. Otherwise, ignore this message.
FunctionResult successResult = new FunctionResult(true);
BlobQueueRegistration registration;
if (!_registrations.TryGetValue(functionId, out registration))
{
return(successResult);
}
IStorageBlobContainer container = registration.BlobClient.GetContainerReference(message.ContainerName);
string blobName = message.BlobName;
IStorageBlob blob;
switch (message.BlobType)
{
case StorageBlobType.PageBlob:
blob = container.GetPageBlobReference(blobName);
break;
case StorageBlobType.BlockBlob:
default:
blob = container.GetBlockBlobReference(blobName);
break;
}
// Ensure the blob still exists with the same ETag.
string possibleETag = await _eTagReader.GetETagAsync(blob, cancellationToken);
if (possibleETag == null)
{
// If the blob no longer exists, just ignore this message.
return(successResult);
}
// If the blob still exists but the ETag is different, delete the message but do a fast path notification.
if (!String.Equals(message.ETag, possibleETag, StringComparison.Ordinal))
{
_blobWrittenWatcher.Notify(blob);
return(successResult);
}
//// If the blob still exists and its ETag is still valid, execute.
//// Note: it's possible the blob could change/be deleted between now and when the function executes.
Guid?parentId = await _causalityReader.GetWriterAsync(blob, cancellationToken);
TriggeredFunctionData input = new TriggeredFunctionData
{
ParentId = parentId,
TriggerValue = blob
};
return(await registration.Executor.TryExecuteAsync(input, cancellationToken));
}