public ScenarioTests(AzuriteFixture azuriteFixture) { account = azuriteFixture.GetAccount(); account.CreateBlobServiceClient().GetBlobContainerClient(ContainerName).DeleteIfExists(); // make sure our system containers are present account.CreateBlobServiceClient().GetBlobContainerClient("azure-webjobs-hosts").CreateIfNotExists(); account.CreateQueueServiceClient().GetQueueClient(QueueName).DeleteIfExists(); }
public void SetUp() { account = AzuriteNUnitFixture.Instance.GetAccount(); account.CreateBlobServiceClient().GetBlobContainerClient(ContainerName).DeleteIfExists(); // make sure our system containers are present account.CreateBlobServiceClient().GetBlobContainerClient("azure-webjobs-hosts").CreateIfNotExists(); account.CreateQueueServiceClient().GetQueueClient(QueueName).DeleteIfExists(); }
public async Task LoadLatestScan_NoContainer_ReturnsNull() { string hostId = Guid.NewGuid().ToString(); string storageAccountName = Guid.NewGuid().ToString(); string containerName = Guid.NewGuid().ToString(); var client = account.CreateBlobServiceClient(); // by default there is no table in this client var manager = new StorageBlobScanInfoManager(hostId, client); var result = await manager.LoadLatestScanAsync(storageAccountName, containerName); Assert.Null(result); }
public BlobTriggerBinding(ParameterInfo parameter, StorageAccount hostAccount, StorageAccount dataAccount, IBlobPathSource path, IHostIdProvider hostIdProvider, QueuesOptions queueOptions, BlobsOptions blobsOptions, IWebJobsExceptionHandler exceptionHandler, IContextSetter <IBlobWrittenWatcher> blobWrittenWatcherSetter, SharedQueueWatcher messageEnqueuedWatcherSetter, ISharedContextProvider sharedContextProvider, IHostSingletonManager singletonManager, ILoggerFactory loggerFactory) { _parameter = parameter ?? throw new ArgumentNullException(nameof(parameter)); _hostAccount = hostAccount ?? throw new ArgumentNullException(nameof(hostAccount)); _dataAccount = dataAccount ?? throw new ArgumentNullException(nameof(dataAccount)); _blobClient = dataAccount.CreateBlobServiceClient(); _accountName = _blobClient.AccountName; _path = path ?? throw new ArgumentNullException(nameof(path)); _hostIdProvider = hostIdProvider ?? throw new ArgumentNullException(nameof(hostIdProvider)); _queueOptions = queueOptions ?? throw new ArgumentNullException(nameof(queueOptions)); _blobsOptions = blobsOptions ?? throw new ArgumentNullException(nameof(blobsOptions)); _exceptionHandler = exceptionHandler ?? throw new ArgumentNullException(nameof(exceptionHandler)); _blobWrittenWatcherSetter = blobWrittenWatcherSetter ?? throw new ArgumentNullException(nameof(blobWrittenWatcherSetter)); _messageEnqueuedWatcherSetter = messageEnqueuedWatcherSetter ?? throw new ArgumentNullException(nameof(messageEnqueuedWatcherSetter)); _sharedContextProvider = sharedContextProvider ?? throw new ArgumentNullException(nameof(sharedContextProvider)); _singletonManager = singletonManager ?? throw new ArgumentNullException(nameof(singletonManager)); _loggerFactory = loggerFactory; _converter = CreateConverter(_blobClient); _bindingDataContract = CreateBindingDataContract(path); }
public BlobTriggerTests(AzuriteFixture azuriteFixture) { account = azuriteFixture.GetAccount(); account.CreateBlobServiceClient().GetBlobContainerClient(ContainerName).DeleteIfExists(); // make sure our system containers are present CreateContainer(account, "azure-webjobs-hosts"); }
private static BlobContainerClient CreateContainer(StorageAccount account, string containerName) { var client = account.CreateBlobServiceClient(); var container = client.GetBlobContainerClient(containerName); container.CreateIfNotExistsAsync().GetAwaiter().GetResult(); return(container); }
private static async Task <BlobContainerClient> CreateContainerAsync(StorageAccount account, string containerName) { var client = account.CreateBlobServiceClient(); var container = client.GetBlobContainerClient(containerName); await container.CreateIfNotExistsAsync(); return(container); }
public BlobTriggerExecutorTests(AzuriteFixture azuriteFixture) { account = azuriteFixture.GetAccount(); account.CreateBlobServiceClient().GetBlobContainerClient(ContainerName).DeleteIfExists(); var loggerFactory = new LoggerFactory(); loggerFactory.AddProvider(_loggerProvider); _logger = loggerFactory.CreateLogger <BlobListener>(); }
public void SetUp() { _loggerProvider = new TestLoggerProvider(); _account = AzuriteNUnitFixture.Instance.GetAccount(); _account.CreateBlobServiceClient().GetBlobContainerClient(ContainerName).DeleteIfExists(); var loggerFactory = new LoggerFactory(); loggerFactory.AddProvider(_loggerProvider); _logger = loggerFactory.CreateLogger <BlobListener>(); }
public async Task BlobChainTest() { // write the initial trigger blob to start the chain var blobClient = _storageAccount.CreateBlobServiceClient(); var container = blobClient.GetBlobContainerClient(_nameResolver.ResolveInString(BlobChainContainerName)); await container.CreateIfNotExistsAsync(); var blob = container.GetBlockBlobClient(BlobChainTriggerBlobName); await blob.UploadTextAsync("0"); var prog = new BlobChainTest_Program(); var host = NewBuilder(prog).Build(); using (prog._completedEvent = new ManualResetEvent(initialState: false)) using (host) { host.Start(); Assert.True(prog._completedEvent.WaitOne(TimeSpan.FromSeconds(60))); } }
public void ExecuteAsync_IfBlobDoesNotMatchPattern_ReturnsSuccessfulResult() { // Arrange var client = account.CreateBlobServiceClient(); string containerName = ContainerName; var container = client.GetBlobContainerClient(containerName); var otherContainer = client.GetBlobContainerClient("other"); IBlobPathSource input = BlobPathSource.Create(containerName + "/{name}"); ITriggerExecutor <BlobTriggerExecutorContext> product = CreateProductUnderTest(input); // Note: this test does not set the PollId. This ensures that we work okay with null values for these. var blob = otherContainer.GetBlockBlobClient("nonmatch"); var context = new BlobTriggerExecutorContext { Blob = new BlobWithContainer <BlobBaseClient>(otherContainer, blob), TriggerSource = BlobTriggerSource.ContainerScan }; // Act Task <FunctionResult> task = product.ExecuteAsync(context, CancellationToken.None); // Assert Assert.True(task.Result.Succeeded); // Validate log is written var logMessage = _loggerProvider.GetAllLogMessages().Single(); Assert.Equal("BlobDoesNotMatchPattern", logMessage.EventId.Name); Assert.Equal(LogLevel.Debug, logMessage.Level); Assert.Equal(6, logMessage.State.Count()); Assert.Equal("FunctionIdLogName", logMessage.GetStateValue <string>("functionName")); Assert.Equal(containerName + "/{name}", logMessage.GetStateValue <string>("pattern")); Assert.Equal(blob.Name, logMessage.GetStateValue <string>("blobName")); Assert.Null(logMessage.GetStateValue <string>("pollId")); Assert.Equal(context.TriggerSource, logMessage.GetStateValue <BlobTriggerSource>("triggerSource")); Assert.True(!string.IsNullOrWhiteSpace(logMessage.GetStateValue <string>("{OriginalFormat}"))); }
public async Task VerifyLockState(string lockId, LeaseState state, LeaseStatus status) { var blobClient = StorageAccount.CreateBlobServiceClient(); var container = blobClient.GetBlobContainerClient("azure-webjobs-hosts"); string blobName = string.Format("locks/{0}/{1}", HostId, lockId); var lockBlob = container.GetBlockBlobClient(blobName); Assert.True(await lockBlob.ExistsAsync()); BlobProperties blobProperties = await lockBlob.GetPropertiesAsync(); Assert.Equal(state, blobProperties.LeaseState); Assert.Equal(status, blobProperties.LeaseStatus); }
public async Task DisposeAsync() { if (Host != null) { await Host.StopAsync(); // $$$ reenalbe this VerifyLockState("WebJobs.Internal.Blobs.Listener", LeaseState.Available, LeaseStatus.Unlocked).Wait(); var blobClient = StorageAccount.CreateBlobServiceClient(); await foreach (var testContainer in blobClient.GetBlobContainersAsync(prefix: TestArtifactPrefix)) { await blobClient.GetBlobContainerClient(testContainer.Name).DeleteAsync(); } } }
private static async Task Clean(StorageAccount account) { var blobClient = account.CreateBlobServiceClient(); await foreach (var testContainer in blobClient.GetBlobContainersAsync(prefix: TestArtifactPrefix)) { await blobClient.GetBlobContainerClient(testContainer.Name).DeleteAsync(); } var queueClient = account.CreateQueueServiceClient(); await foreach (var queue in queueClient.GetQueuesAsync(prefix: TestArtifactPrefix)) { await queueClient.GetQueueClient(queue.Name).DeleteAsync(); } }
public BlobTriggerEndToEndTests() { _nameResolver = new RandomNameResolver(); // pull from a default host var host = new HostBuilder() .ConfigureDefaultTestHost(b => { b.AddAzureStorageBlobs().AddAzureStorageQueues(); }) .Build(); var provider = host.Services.GetService <StorageAccountProvider>(); _storageAccount = provider.GetHost(); var blobClient = _storageAccount.CreateBlobServiceClient(); _testContainer = blobClient.GetBlobContainerClient(_nameResolver.ResolveInString(SingleTriggerContainerName)); Assert.False(_testContainer.ExistsAsync().Result); _testContainer.CreateAsync().Wait(); }
private static IBlobListenerStrategy CreateStrategy(string hostId, StorageAccount account, IWebJobsExceptionHandler exceptionHandler, ILogger <BlobListener> logger) { if (!account.IsDevelopmentStorageAccount()) { IBlobScanInfoManager scanInfoManager = new StorageBlobScanInfoManager(hostId, account.CreateBlobServiceClient()); return(new ScanBlobScanLogHybridPollingStrategy(scanInfoManager, exceptionHandler, logger)); } else { return(new ScanContainersStrategy()); } }
private static BlobContainerClient GetContainerReference(StorageAccount account, string containerName) { var client = account.CreateBlobServiceClient(); return(client.GetBlobContainerClient(ContainerName)); }
public void SetUp() { account = AzuriteNUnitFixture.Instance.GetAccount(); account.CreateBlobServiceClient().GetBlobContainerClient(ContainerName).DeleteIfExists(); account.CreateQueueServiceClient().GetQueueClient(TriggerQueueName).DeleteIfExists(); }
public async Task <IListener> CreateAsync(CancellationToken cancellationToken) { // Note that these clients are intentionally for the storage account rather than for the dashboard account. // We use the storage, not dashboard, account for the blob receipt container and blob trigger queues. var primaryQueueClient = _hostAccount.CreateQueueServiceClient(); var primaryBlobClient = _hostAccount.CreateBlobServiceClient(); // Important: We're using the storage account of the function target here, which is the account that the // function the listener is for is targeting. This is the account that will be used // to read the trigger blob. var targetBlobClient = _dataAccount.CreateBlobServiceClient(); var targetQueueClient = _dataAccount.CreateQueueServiceClient(); string hostId = await _hostIdProvider.GetHostIdAsync(cancellationToken).ConfigureAwait(false); string hostBlobTriggerQueueName = HostQueueNames.GetHostBlobTriggerQueueName(hostId); var hostBlobTriggerQueue = primaryQueueClient.GetQueueClient(hostBlobTriggerQueueName); SharedQueueWatcher sharedQueueWatcher = _messageEnqueuedWatcherSetter; SharedBlobListener sharedBlobListener = _sharedContextProvider.GetOrCreateInstance <SharedBlobListener>( new SharedBlobListenerFactory(hostId, _hostAccount, _exceptionHandler, _blobWrittenWatcherSetter, _loggerFactory.CreateLogger <BlobListener>())); // Register the blob container we wish to monitor with the shared blob listener. await RegisterWithSharedBlobListenerAsync(hostId, sharedBlobListener, primaryBlobClient, hostBlobTriggerQueue, sharedQueueWatcher, cancellationToken).ConfigureAwait(false); // Create a "bridge" listener that will monitor the blob // notification queue and dispatch to the target job function. SharedBlobQueueListener sharedBlobQueueListener = _sharedContextProvider.GetOrCreateInstance <SharedBlobQueueListener>( new SharedBlobQueueListenerFactory(_hostAccount, sharedQueueWatcher, hostBlobTriggerQueue, _queueOptions, _exceptionHandler, _loggerFactory, sharedBlobListener.BlobWritterWatcher, _functionDescriptor)); var queueListener = new BlobListener(sharedBlobQueueListener); // determine which client to use for the poison queue // by default this should target the same storage account // as the blob container we're monitoring var poisonQueueClient = targetQueueClient; if ( // _dataAccount.Type != StorageAccountType.GeneralPurpose || $$$ _blobsOptions.CentralizedPoisonQueue) { // use the primary storage account if the centralize flag is true, // or if the target storage account doesn't support queues poisonQueueClient = primaryQueueClient; } // Register our function with the shared blob queue listener RegisterWithSharedBlobQueueListenerAsync(sharedBlobQueueListener, targetBlobClient, poisonQueueClient); // check a flag in the shared context to see if we've created the singleton // shared blob listener in this host instance object singletonListenerCreated = false; if (!_sharedContextProvider.TryGetValue(SingletonBlobListenerScopeId, out singletonListenerCreated)) { // Create a singleton shared blob listener, since we only // want a single instance of the blob poll/scan logic to be running // across host instances var singletonBlobListener = _singletonManager.CreateHostSingletonListener( new BlobListener(sharedBlobListener), SingletonBlobListenerScopeId); _sharedContextProvider.SetValue(SingletonBlobListenerScopeId, true); return(new CompositeListener(singletonBlobListener, queueListener)); } else { // We've already created the singleton blob listener // so just return our queue listener. Note that while we want the // blob scan to be singleton, the shared queue listener needs to run // on ALL instances so load can be scaled out return(queueListener); } }
public async Task InitializeAsync() { // TODO (kasobol-msft) find better way string connectionString = Environment.GetEnvironmentVariable("AzureWebJobsStorage"); if (!string.IsNullOrWhiteSpace(connectionString)) { RandomNameResolver nameResolver = new RandomNameResolver(); Host = new HostBuilder() .ConfigureDefaultTestHost <BlobBindingEndToEndTests>(b => { b.AddAzureStorageBlobs().AddAzureStorageQueues(); b.AddAzureStorageCoreServices(); }) .ConfigureServices(services => { services.AddSingleton <INameResolver>(nameResolver); }) .Build(); JobHost = Host.GetJobHost(); var provider = Host.Services.GetService <StorageAccountProvider>(); StorageAccount = provider.GetHost(); var blobClient = StorageAccount.CreateBlobServiceClient(); BlobContainer = blobClient.GetBlobContainerClient(nameResolver.ResolveInString(ContainerName)); Assert.False(await BlobContainer.ExistsAsync()); await BlobContainer.CreateAsync(); OutputBlobContainer = blobClient.GetBlobContainerClient(nameResolver.ResolveInString(OutputContainerName)); var pageBlobContainer = blobClient.GetBlobContainerClient(nameResolver.ResolveInString(PageBlobContainerName)); Assert.False(await pageBlobContainer.ExistsAsync()); await pageBlobContainer.CreateAsync(); var hierarchicalBlobContainer = blobClient.GetBlobContainerClient(nameResolver.ResolveInString(HierarchicalBlobContainerName)); Assert.False(await hierarchicalBlobContainer.ExistsAsync()); await hierarchicalBlobContainer.CreateAsync(); var appendBlobContainer = blobClient.GetBlobContainerClient(nameResolver.ResolveInString(AppendBlobContainerName)); Assert.False(await appendBlobContainer.ExistsAsync()); await appendBlobContainer.CreateAsync(); await Host.StartAsync(); // upload some test blobs BlockBlobClient blob = BlobContainer.GetBlockBlobClient("blob1"); await blob.UploadTextAsync(TestData); blob = BlobContainer.GetBlockBlobClient("blob2"); await blob.UploadTextAsync(TestData); blob = BlobContainer.GetBlockBlobClient("blob3"); await blob.UploadTextAsync(TestData); blob = BlobContainer.GetBlockBlobClient("file1"); await blob.UploadTextAsync(TestData); blob = BlobContainer.GetBlockBlobClient("file2"); await blob.UploadTextAsync(TestData); blob = BlobContainer.GetBlockBlobClient("overwrite"); await blob.UploadTextAsync(TestData); // add a couple hierarchical blob paths blob = hierarchicalBlobContainer.GetBlockBlobClient("sub/blob1"); await blob.UploadTextAsync(TestData); blob = hierarchicalBlobContainer.GetBlockBlobClient("sub/blob2"); await blob.UploadTextAsync(TestData); blob = hierarchicalBlobContainer.GetBlockBlobClient("sub/sub/blob3"); await blob.UploadTextAsync(TestData); blob = hierarchicalBlobContainer.GetBlockBlobClient("blob4"); await blob.UploadTextAsync(TestData); byte[] bytes = new byte[512]; byte[] testBytes = Encoding.UTF8.GetBytes(TestData); for (int i = 0; i < testBytes.Length; i++) { bytes[i] = testBytes[i]; } PageBlobClient pageBlob = pageBlobContainer.GetPageBlobClient("blob1"); await pageBlob.UploadFromByteArrayAsync(bytes, 0); pageBlob = pageBlobContainer.GetPageBlobClient("blob2"); await pageBlob.UploadFromByteArrayAsync(bytes, 0); AppendBlobClient appendBlob = appendBlobContainer.GetAppendBlobClient("blob1"); await appendBlob.UploadTextAsync(TestData); appendBlob = appendBlobContainer.GetAppendBlobClient("blob2"); await appendBlob.UploadTextAsync(TestData); appendBlob = appendBlobContainer.GetAppendBlobClient("blob3"); await appendBlob.UploadTextAsync(TestData); } }
public BlobTests(AzuriteFixture azuriteFixture) { account = azuriteFixture.GetAccount(); account.CreateBlobServiceClient().GetBlobContainerClient(ContainerName).DeleteIfExists(); account.CreateQueueServiceClient().GetQueueClient(TriggerQueueName).DeleteIfExists(); }
public HostCallTests(AzuriteFixture azuriteFixture) { account = azuriteFixture.GetAccount(); account.CreateBlobServiceClient().GetBlobContainerClient(ContainerName).DeleteIfExists(); }