/// <inheritdoc /> public async Task <string> EnqueueAsync(T message, CancellationToken cancellationToken) { await _blobContainer.CreateIfNotExistsAsync(cancellationToken); string blobName = BlobNames.GetConflictFreeDateTimeBasedBlobName(); IStorageBlockBlob blob = _blobContainer.GetBlockBlobReference(blobName); message.AddMetadata(blob.Metadata); string messageBody = JsonConvert.SerializeObject(message, JsonSerialization.Settings); await blob.UploadTextAsync(messageBody, cancellationToken : cancellationToken); return(blobName); }
public void BlobTrigger_IfBindingAlwaysFails_MovesToPoisonQueue() { // Arrange IStorageAccount account = CreateFakeStorageAccount(); IStorageBlobContainer container = CreateContainer(account, ContainerName); IStorageBlockBlob blob = container.GetBlockBlobReference(BlobName); CloudBlockBlob expectedBlob = blob.SdkObject; blob.UploadText("ignore"); // Act string result = RunTrigger <string>(account, typeof(PoisonBlobProgram), (s) => PoisonBlobProgram.TaskSource = s, new string[] { typeof(PoisonBlobProgram).FullName + ".PutInPoisonQueue" }); // Assert BlobTriggerMessage message = JsonConvert.DeserializeObject <BlobTriggerMessage>(result); Assert.NotNull(message); Assert.Equal(typeof(PoisonBlobProgram).FullName + ".PutInPoisonQueue", message.FunctionId); Assert.Equal(StorageBlobType.BlockBlob, message.BlobType); Assert.Equal(ContainerName, message.ContainerName); Assert.Equal(BlobName, message.BlobName); Assert.NotEmpty(message.ETag); }
public async Task <IEnumerable <IStorageBlob> > GetRecentBlobWritesAsync(CancellationToken cancellationToken, int hoursWindow = DefaultScanHoursWindow) { List <IStorageBlob> blobs = new List <IStorageBlob>(); var time = DateTime.UtcNow; // will scan back 2 hours, which is enough to deal with clock sqew foreach (var blob in await ListRecentLogFilesAsync(_blobClient, time, cancellationToken, hoursWindow)) { bool isAdded = _scannedBlobNames.Add(blob.Name); if (!isAdded) { continue; } // Need to clear out cache. if (_scannedBlobNames.Count > 100 * 1000) { _scannedBlobNames.Clear(); } IEnumerable <StorageAnalyticsLogEntry> entries = await _parser.ParseLogAsync(blob, cancellationToken); IEnumerable <BlobPath> filteredBlobs = GetPathsForValidBlobWrites(entries); foreach (BlobPath path in filteredBlobs) { IStorageBlobContainer container = _blobClient.GetContainerReference(path.ContainerName); blobs.Add(container.GetBlockBlobReference(path.BlobName)); } } return(blobs); }
public async Task <IEnumerable <IStorageBlob> > GetRecentBlobWritesAsync(CancellationToken cancellationToken, int hoursWindow = DefaultScanHoursWindow) { List <IStorageBlob> blobs = new List <IStorageBlob>(); var time = DateTime.UtcNow; // will scan back 2 hours, which is enough to deal with clock sqew foreach (var blob in await ListRecentLogFilesAsync(_blobClient, time, cancellationToken, hoursWindow)) { bool isAdded = _scannedBlobNames.Add(blob.Name); if (!isAdded) { continue; } // Need to clear out cache. if (_scannedBlobNames.Count > 100 * 1000) { _scannedBlobNames.Clear(); } var parsedBlobPaths = from entry in await _parser.ParseLogAsync(blob, cancellationToken) where entry.IsBlobWrite select entry.ToBlobPath(); foreach (BlobPath path in parsedBlobPaths.Where(p => p != null)) { IStorageBlobContainer container = _blobClient.GetContainerReference(path.ContainerName); blobs.Add(container.GetBlockBlobReference(path.BlobName)); } } return(blobs); }
private static IStorageBlob CreateBlobReference(string containerName, string blobName) { IStorageAccount account = CreateAccount(); IStorageBlobClient client = account.CreateBlobClient(); IStorageBlobContainer container = client.GetContainerReference(containerName); return(container.GetBlockBlobReference(blobName)); }
private string CreateAblobAndUploadToContainer(IStorageBlobContainer container, string blobContent = "test") { string blobName = Path.GetRandomFileName().Replace(".", ""); IStorageBlockBlob blob = container.GetBlockBlobReference(blobName); container.CreateIfNotExists(); blob.UploadText(blobContent); return(blobName); }
public void BlobTriggerToQueueTriggerToBlob_WritesFinalBlob() { // Arrange IStorageAccount account = CreateFakeStorageAccount(); IStorageBlobContainer container = CreateContainer(account, ContainerName); IStorageBlockBlob inputBlob = container.GetBlockBlobReference(BlobName); inputBlob.UploadText("15"); // Act RunTrigger <object>(account, typeof(BlobTriggerToQueueTriggerToBlobProgram), (s) => BlobTriggerToQueueTriggerToBlobProgram.TaskSource = s); // Assert IStorageBlockBlob outputBlob = container.GetBlockBlobReference(OutputBlobName); string content = outputBlob.DownloadText(); Assert.Equal("16", content); }
public void BlobTrigger_IfWritesToSecondBlobTrigger_TriggersOutputQuickly() { // Arrange IStorageAccount account = CreateFakeStorageAccount(); IStorageBlobContainer container = CreateContainer(account, ContainerName); IStorageBlockBlob inputBlob = container.GetBlockBlobReference(BlobName); inputBlob.UploadText("abc"); // Act RunTrigger <object>(account, typeof(BlobTriggerToBlobTriggerProgram), (s) => BlobTriggerToBlobTriggerProgram.TaskSource = s); // Assert IStorageBlockBlob outputBlob = container.GetBlockBlobReference(OutputBlobName); string content = outputBlob.DownloadText(); Assert.Equal("*abc*", content); }
public void BlobTrigger_Binding_Metadata() { var app = new BindToCloudBlob2Program(); var activator = new FakeActivator(app); IStorageAccount account = CreateFakeStorageAccount(); var host = TestHelpers.NewJobHost <BindToCloudBlob2Program>(account, activator); // Set the binding data, and verify it's accessible in the function. IStorageBlobContainer container = CreateContainer(account, ContainerName); IStorageBlockBlob blob = container.GetBlockBlobReference(BlobName); blob.Metadata["m1"] = "v1"; host.Call("Run", new { blob = blob }); Assert.True(app.Success); }
public void BlobTrigger_IfBoundToCloudBlob_Binds() { // Arrange IStorageAccount account = CreateFakeStorageAccount(); IStorageBlobContainer container = CreateContainer(account, ContainerName); IStorageBlockBlob blob = container.GetBlockBlobReference(BlobName); CloudBlockBlob expectedBlob = blob.SdkObject; blob.UploadText("ignore"); // Act ICloudBlob result = RunTrigger <ICloudBlob>(account, typeof(BindToCloudBlobProgram), (s) => BindToCloudBlobProgram.TaskSource = s); // Assert Assert.Equal(expectedBlob.Uri, result.Uri); }
public static Task <IStorageBlob> GetBlobReferenceForArgumentTypeAsync(this IStorageBlobContainer container, string blobName, Type argumentType, CancellationToken cancellationToken) { if (argumentType == typeof(CloudBlockBlob)) { IStorageBlob blob = container.GetBlockBlobReference(blobName); return(Task.FromResult(blob)); } else if (argumentType == typeof(CloudPageBlob)) { IStorageBlob blob = container.GetPageBlobReference(blobName); return(Task.FromResult(blob)); } else { return(GetExistingOrNewBlockBlobReferenceAsync(container, blobName, cancellationToken)); } }
public void TestBlobListener() { const string containerName = "container"; IStorageAccount account = CreateFakeStorageAccount(); IStorageBlobContainer container = account.CreateBlobClient().GetContainerReference(containerName); IBlobListenerStrategy product = new ScanContainersStrategy(); LambdaBlobTriggerExecutor executor = new LambdaBlobTriggerExecutor(); product.Register(container, executor); product.Start(); executor.ExecuteLambda = (_) => { throw new InvalidOperationException("shouldn't be any blobs in the container"); }; product.Execute(); const string expectedBlobName = "foo1.csv"; IStorageBlockBlob blob = container.GetBlockBlobReference(expectedBlobName); container.CreateIfNotExists(); blob.UploadText("ignore"); int count = 0; executor.ExecuteLambda = (b) => { count++; Assert.Equal(expectedBlobName, b.Name); return(true); }; product.Execute(); Assert.Equal(1, count); // Now run again; shouldn't show up. executor.ExecuteLambda = (_) => { throw new InvalidOperationException("shouldn't retrigger the same blob"); }; product.Execute(); }
public void ExecuteAsync_IfBlobDoesNotMatchPattern_ReturnsTrue() { // Arrange IStorageAccount account = CreateAccount(); IStorageBlobClient client = account.CreateBlobClient(); string containerName = "container"; IStorageBlobContainer container = client.GetContainerReference(containerName); IStorageBlobContainer otherContainer = client.GetContainerReference("other"); IBlobPathSource input = BlobPathSource.Create(containerName + "/{name}"); ITriggerExecutor <IStorageBlob> product = CreateProductUnderTest(input); IStorageBlob blob = otherContainer.GetBlockBlobReference("nonmatch"); // Act Task <bool> task = product.ExecuteAsync(blob, CancellationToken.None); // Assert Assert.True(task.Result); }
private static async Task <IStorageBlob> GetExistingOrNewBlockBlobReferenceAsync(IStorageBlobContainer container, string blobName, CancellationToken cancellationToken) { try { return(await container.GetBlobReferenceFromServerAsync(blobName, cancellationToken)); } catch (StorageException exception) { RequestResult result = exception.RequestInformation; if (result == null || result.HttpStatusCode != 404) { throw; } else { return(container.GetBlockBlobReference(blobName)); } } }
private static async Task<IStorageBlob> GetExistingOrNewBlockBlobReferenceAsync(IStorageBlobContainer container, string blobName, CancellationToken cancellationToken) { try { return await container.GetBlobReferenceFromServerAsync(blobName, cancellationToken); } catch (StorageException exception) { RequestResult result = exception.RequestInformation; if (result == null || result.HttpStatusCode != 404) { throw; } else { return container.GetBlockBlobReference(blobName); } } }
public void Blob_IfBoundToTextWriter_CreatesBlob() { // Arrange const string expectedContent = "message"; IStorageAccount account = CreateFakeStorageAccount(); IStorageQueue triggerQueue = CreateQueue(account, TriggerQueueName); triggerQueue.AddMessage(triggerQueue.CreateMessage(expectedContent)); // Act RunTrigger(account, typeof(BindToTextWriterProgram)); // Assert IStorageBlobContainer container = GetContainerReference(account, ContainerName); Assert.True(container.Exists()); IStorageBlockBlob blob = container.GetBlockBlobReference(BlobName); Assert.True(blob.Exists()); string content = blob.DownloadText(); Assert.Equal(expectedContent, content); }
public void Blob_IfBoundToCloudBlockBlob_BindsAndCreatesContainerButNotBlob() { // Arrange IStorageAccount account = CreateFakeStorageAccount(); IStorageQueue triggerQueue = CreateQueue(account, TriggerQueueName); triggerQueue.AddMessage(triggerQueue.CreateMessage("ignore")); // Act CloudBlockBlob result = RunTrigger <CloudBlockBlob>(account, typeof(BindToCloudBlockBlobProgram), (s) => BindToCloudBlockBlobProgram.TaskSource = s); // Assert Assert.NotNull(result); Assert.Equal(BlobName, result.Name); Assert.NotNull(result.Container); Assert.Equal(ContainerName, result.Container.Name); IStorageBlobContainer container = GetContainerReference(account, ContainerName); Assert.True(container.Exists()); IStorageBlockBlob blob = container.GetBlockBlobReference(BlobName); Assert.False(blob.Exists()); }
private IStorageBlockBlob GetBlockBlobReference(LocalBlobDescriptor descriptor) { IStorageBlobContainer container = _client.GetContainerReference(descriptor.ContainerName); return(container.GetBlockBlobReference(descriptor.BlobName)); }
public async Task <FunctionResult> ExecuteAsync(IStorageQueueMessage value, CancellationToken cancellationToken) { BlobTriggerMessage message = JsonConvert.DeserializeObject <BlobTriggerMessage>(value.AsString, JsonSerialization.Settings); if (message == null) { throw new InvalidOperationException("Invalid blob trigger message."); } string functionId = message.FunctionId; if (functionId == null) { throw new InvalidOperationException("Invalid function ID."); } // Ensure that the function ID is still valid. Otherwise, ignore this message. FunctionResult successResult = new FunctionResult(true); BlobQueueRegistration registration; if (!_registrations.TryGetValue(functionId, out registration)) { return(successResult); } IStorageBlobContainer container = registration.BlobClient.GetContainerReference(message.ContainerName); string blobName = message.BlobName; IStorageBlob blob; switch (message.BlobType) { case StorageBlobType.PageBlob: blob = container.GetPageBlobReference(blobName); break; case StorageBlobType.BlockBlob: default: blob = container.GetBlockBlobReference(blobName); break; } // Ensure the blob still exists with the same ETag. string possibleETag = await _eTagReader.GetETagAsync(blob, cancellationToken); if (possibleETag == null) { // If the blob no longer exists, just ignore this message. return(successResult); } // If the blob still exists but the ETag is different, delete the message but do a fast path notification. if (!String.Equals(message.ETag, possibleETag, StringComparison.Ordinal)) { _blobWrittenWatcher.Notify(blob); return(successResult); } //// If the blob still exists and its ETag is still valid, execute. //// Note: it's possible the blob could change/be deleted between now and when the function executes. Guid?parentId = await _causalityReader.GetWriterAsync(blob, cancellationToken); TriggeredFunctionData input = new TriggeredFunctionData { ParentId = parentId, TriggerValue = blob }; return(await registration.Executor.TryExecuteAsync(input, cancellationToken)); }