public ScenarioTests(AzuriteFixture azuriteFixture)
{
account = azuriteFixture.GetAccount();
account.CreateBlobServiceClient().GetBlobContainerClient(ContainerName).DeleteIfExists();
// make sure our system containers are present
account.CreateBlobServiceClient().GetBlobContainerClient("azure-webjobs-hosts").CreateIfNotExists();
account.CreateQueueServiceClient().GetQueueClient(QueueName).DeleteIfExists();
}
public void SetUp()
{
account = AzuriteNUnitFixture.Instance.GetAccount();
account.CreateBlobServiceClient().GetBlobContainerClient(ContainerName).DeleteIfExists();
// make sure our system containers are present
account.CreateBlobServiceClient().GetBlobContainerClient("azure-webjobs-hosts").CreateIfNotExists();
account.CreateQueueServiceClient().GetQueueClient(QueueName).DeleteIfExists();
}
public async Task LoadLatestScan_NoContainer_ReturnsNull()
{
string hostId = Guid.NewGuid().ToString();
string storageAccountName = Guid.NewGuid().ToString();
string containerName = Guid.NewGuid().ToString();
var client = account.CreateBlobServiceClient();
// by default there is no table in this client
var manager = new StorageBlobScanInfoManager(hostId, client);
var result = await manager.LoadLatestScanAsync(storageAccountName, containerName);
Assert.Null(result);
}
public BlobTriggerBinding(ParameterInfo parameter,
StorageAccount hostAccount,
StorageAccount dataAccount,
IBlobPathSource path,
IHostIdProvider hostIdProvider,
QueuesOptions queueOptions,
BlobsOptions blobsOptions,
IWebJobsExceptionHandler exceptionHandler,
IContextSetter <IBlobWrittenWatcher> blobWrittenWatcherSetter,
SharedQueueWatcher messageEnqueuedWatcherSetter,
ISharedContextProvider sharedContextProvider,
IHostSingletonManager singletonManager,
ILoggerFactory loggerFactory)
{
_parameter = parameter ?? throw new ArgumentNullException(nameof(parameter));
_hostAccount = hostAccount ?? throw new ArgumentNullException(nameof(hostAccount));
_dataAccount = dataAccount ?? throw new ArgumentNullException(nameof(dataAccount));
_blobClient = dataAccount.CreateBlobServiceClient();
_accountName = _blobClient.AccountName;
_path = path ?? throw new ArgumentNullException(nameof(path));
_hostIdProvider = hostIdProvider ?? throw new ArgumentNullException(nameof(hostIdProvider));
_queueOptions = queueOptions ?? throw new ArgumentNullException(nameof(queueOptions));
_blobsOptions = blobsOptions ?? throw new ArgumentNullException(nameof(blobsOptions));
_exceptionHandler = exceptionHandler ?? throw new ArgumentNullException(nameof(exceptionHandler));
_blobWrittenWatcherSetter = blobWrittenWatcherSetter ?? throw new ArgumentNullException(nameof(blobWrittenWatcherSetter));
_messageEnqueuedWatcherSetter = messageEnqueuedWatcherSetter ?? throw new ArgumentNullException(nameof(messageEnqueuedWatcherSetter));
_sharedContextProvider = sharedContextProvider ?? throw new ArgumentNullException(nameof(sharedContextProvider));
_singletonManager = singletonManager ?? throw new ArgumentNullException(nameof(singletonManager));
_loggerFactory = loggerFactory;
_converter = CreateConverter(_blobClient);
_bindingDataContract = CreateBindingDataContract(path);
}
public BlobTriggerTests(AzuriteFixture azuriteFixture)
{
account = azuriteFixture.GetAccount();
account.CreateBlobServiceClient().GetBlobContainerClient(ContainerName).DeleteIfExists();
// make sure our system containers are present
CreateContainer(account, "azure-webjobs-hosts");
}
private static BlobContainerClient CreateContainer(StorageAccount account, string containerName)
{
var client = account.CreateBlobServiceClient();
var container = client.GetBlobContainerClient(containerName);
container.CreateIfNotExistsAsync().GetAwaiter().GetResult();
return(container);
}
private static async Task <BlobContainerClient> CreateContainerAsync(StorageAccount account, string containerName)
{
var client = account.CreateBlobServiceClient();
var container = client.GetBlobContainerClient(containerName);
await container.CreateIfNotExistsAsync();
return(container);
}
public BlobTriggerExecutorTests(AzuriteFixture azuriteFixture)
{
account = azuriteFixture.GetAccount();
account.CreateBlobServiceClient().GetBlobContainerClient(ContainerName).DeleteIfExists();
var loggerFactory = new LoggerFactory();
loggerFactory.AddProvider(_loggerProvider);
_logger = loggerFactory.CreateLogger <BlobListener>();
}
public void SetUp()
{
_loggerProvider = new TestLoggerProvider();
_account = AzuriteNUnitFixture.Instance.GetAccount();
_account.CreateBlobServiceClient().GetBlobContainerClient(ContainerName).DeleteIfExists();
var loggerFactory = new LoggerFactory();
loggerFactory.AddProvider(_loggerProvider);
_logger = loggerFactory.CreateLogger <BlobListener>();
}
public async Task BlobChainTest()
{
// write the initial trigger blob to start the chain
var blobClient = _storageAccount.CreateBlobServiceClient();
var container = blobClient.GetBlobContainerClient(_nameResolver.ResolveInString(BlobChainContainerName));
await container.CreateIfNotExistsAsync();
var blob = container.GetBlockBlobClient(BlobChainTriggerBlobName);
await blob.UploadTextAsync("0");
var prog = new BlobChainTest_Program();
var host = NewBuilder(prog).Build();
using (prog._completedEvent = new ManualResetEvent(initialState: false))
using (host)
{
host.Start();
Assert.True(prog._completedEvent.WaitOne(TimeSpan.FromSeconds(60)));
}
}
public void ExecuteAsync_IfBlobDoesNotMatchPattern_ReturnsSuccessfulResult()
{
// Arrange
var client = account.CreateBlobServiceClient();
string containerName = ContainerName;
var container = client.GetBlobContainerClient(containerName);
var otherContainer = client.GetBlobContainerClient("other");
IBlobPathSource input = BlobPathSource.Create(containerName + "/{name}");
ITriggerExecutor <BlobTriggerExecutorContext> product = CreateProductUnderTest(input);
// Note: this test does not set the PollId. This ensures that we work okay with null values for these.
var blob = otherContainer.GetBlockBlobClient("nonmatch");
var context = new BlobTriggerExecutorContext
{
Blob = new BlobWithContainer <BlobBaseClient>(otherContainer, blob),
TriggerSource = BlobTriggerSource.ContainerScan
};
// Act
Task <FunctionResult> task = product.ExecuteAsync(context, CancellationToken.None);
// Assert
Assert.True(task.Result.Succeeded);
// Validate log is written
var logMessage = _loggerProvider.GetAllLogMessages().Single();
Assert.Equal("BlobDoesNotMatchPattern", logMessage.EventId.Name);
Assert.Equal(LogLevel.Debug, logMessage.Level);
Assert.Equal(6, logMessage.State.Count());
Assert.Equal("FunctionIdLogName", logMessage.GetStateValue <string>("functionName"));
Assert.Equal(containerName + "/{name}", logMessage.GetStateValue <string>("pattern"));
Assert.Equal(blob.Name, logMessage.GetStateValue <string>("blobName"));
Assert.Null(logMessage.GetStateValue <string>("pollId"));
Assert.Equal(context.TriggerSource, logMessage.GetStateValue <BlobTriggerSource>("triggerSource"));
Assert.True(!string.IsNullOrWhiteSpace(logMessage.GetStateValue <string>("{OriginalFormat}")));
}
public async Task VerifyLockState(string lockId, LeaseState state, LeaseStatus status)
{
var blobClient = StorageAccount.CreateBlobServiceClient();
var container = blobClient.GetBlobContainerClient("azure-webjobs-hosts");
string blobName = string.Format("locks/{0}/{1}", HostId, lockId);
var lockBlob = container.GetBlockBlobClient(blobName);
Assert.True(await lockBlob.ExistsAsync());
BlobProperties blobProperties = await lockBlob.GetPropertiesAsync();
Assert.Equal(state, blobProperties.LeaseState);
Assert.Equal(status, blobProperties.LeaseStatus);
}
public async Task DisposeAsync()
{
if (Host != null)
{
await Host.StopAsync();
// $$$ reenalbe this
VerifyLockState("WebJobs.Internal.Blobs.Listener", LeaseState.Available, LeaseStatus.Unlocked).Wait();
var blobClient = StorageAccount.CreateBlobServiceClient();
await foreach (var testContainer in blobClient.GetBlobContainersAsync(prefix: TestArtifactPrefix))
{
await blobClient.GetBlobContainerClient(testContainer.Name).DeleteAsync();
}
}
}
private static async Task Clean(StorageAccount account)
{
var blobClient = account.CreateBlobServiceClient();
await foreach (var testContainer in blobClient.GetBlobContainersAsync(prefix: TestArtifactPrefix))
{
await blobClient.GetBlobContainerClient(testContainer.Name).DeleteAsync();
}
var queueClient = account.CreateQueueServiceClient();
await foreach (var queue in queueClient.GetQueuesAsync(prefix: TestArtifactPrefix))
{
await queueClient.GetQueueClient(queue.Name).DeleteAsync();
}
}
public BlobTriggerEndToEndTests()
{
_nameResolver = new RandomNameResolver();
// pull from a default host
var host = new HostBuilder()
.ConfigureDefaultTestHost(b =>
{
b.AddAzureStorageBlobs().AddAzureStorageQueues();
})
.Build();
var provider = host.Services.GetService <StorageAccountProvider>();
_storageAccount = provider.GetHost();
var blobClient = _storageAccount.CreateBlobServiceClient();
_testContainer = blobClient.GetBlobContainerClient(_nameResolver.ResolveInString(SingleTriggerContainerName));
Assert.False(_testContainer.ExistsAsync().Result);
_testContainer.CreateAsync().Wait();
}
private static IBlobListenerStrategy CreateStrategy(string hostId, StorageAccount account, IWebJobsExceptionHandler exceptionHandler, ILogger <BlobListener> logger)
{
if (!account.IsDevelopmentStorageAccount())
{
IBlobScanInfoManager scanInfoManager = new StorageBlobScanInfoManager(hostId, account.CreateBlobServiceClient());
return(new ScanBlobScanLogHybridPollingStrategy(scanInfoManager, exceptionHandler, logger));
}
else
{
return(new ScanContainersStrategy());
}
}
private static BlobContainerClient GetContainerReference(StorageAccount account, string containerName)
{
var client = account.CreateBlobServiceClient();
return(client.GetBlobContainerClient(ContainerName));
}
public void SetUp()
{
account = AzuriteNUnitFixture.Instance.GetAccount();
account.CreateBlobServiceClient().GetBlobContainerClient(ContainerName).DeleteIfExists();
account.CreateQueueServiceClient().GetQueueClient(TriggerQueueName).DeleteIfExists();
}
public async Task <IListener> CreateAsync(CancellationToken cancellationToken)
{
// Note that these clients are intentionally for the storage account rather than for the dashboard account.
// We use the storage, not dashboard, account for the blob receipt container and blob trigger queues.
var primaryQueueClient = _hostAccount.CreateQueueServiceClient();
var primaryBlobClient = _hostAccount.CreateBlobServiceClient();
// Important: We're using the storage account of the function target here, which is the account that the
// function the listener is for is targeting. This is the account that will be used
// to read the trigger blob.
var targetBlobClient = _dataAccount.CreateBlobServiceClient();
var targetQueueClient = _dataAccount.CreateQueueServiceClient();
string hostId = await _hostIdProvider.GetHostIdAsync(cancellationToken).ConfigureAwait(false);
string hostBlobTriggerQueueName = HostQueueNames.GetHostBlobTriggerQueueName(hostId);
var hostBlobTriggerQueue = primaryQueueClient.GetQueueClient(hostBlobTriggerQueueName);
SharedQueueWatcher sharedQueueWatcher = _messageEnqueuedWatcherSetter;
SharedBlobListener sharedBlobListener = _sharedContextProvider.GetOrCreateInstance <SharedBlobListener>(
new SharedBlobListenerFactory(hostId, _hostAccount, _exceptionHandler, _blobWrittenWatcherSetter, _loggerFactory.CreateLogger <BlobListener>()));
// Register the blob container we wish to monitor with the shared blob listener.
await RegisterWithSharedBlobListenerAsync(hostId, sharedBlobListener, primaryBlobClient,
hostBlobTriggerQueue, sharedQueueWatcher, cancellationToken).ConfigureAwait(false);
// Create a "bridge" listener that will monitor the blob
// notification queue and dispatch to the target job function.
SharedBlobQueueListener sharedBlobQueueListener = _sharedContextProvider.GetOrCreateInstance <SharedBlobQueueListener>(
new SharedBlobQueueListenerFactory(_hostAccount, sharedQueueWatcher, hostBlobTriggerQueue,
_queueOptions, _exceptionHandler, _loggerFactory, sharedBlobListener.BlobWritterWatcher, _functionDescriptor));
var queueListener = new BlobListener(sharedBlobQueueListener);
// determine which client to use for the poison queue
// by default this should target the same storage account
// as the blob container we're monitoring
var poisonQueueClient = targetQueueClient;
if (
// _dataAccount.Type != StorageAccountType.GeneralPurpose || $$$
_blobsOptions.CentralizedPoisonQueue)
{
// use the primary storage account if the centralize flag is true,
// or if the target storage account doesn't support queues
poisonQueueClient = primaryQueueClient;
}
// Register our function with the shared blob queue listener
RegisterWithSharedBlobQueueListenerAsync(sharedBlobQueueListener, targetBlobClient, poisonQueueClient);
// check a flag in the shared context to see if we've created the singleton
// shared blob listener in this host instance
object singletonListenerCreated = false;
if (!_sharedContextProvider.TryGetValue(SingletonBlobListenerScopeId, out singletonListenerCreated))
{
// Create a singleton shared blob listener, since we only
// want a single instance of the blob poll/scan logic to be running
// across host instances
var singletonBlobListener = _singletonManager.CreateHostSingletonListener(
new BlobListener(sharedBlobListener), SingletonBlobListenerScopeId);
_sharedContextProvider.SetValue(SingletonBlobListenerScopeId, true);
return(new CompositeListener(singletonBlobListener, queueListener));
}
else
{
// We've already created the singleton blob listener
// so just return our queue listener. Note that while we want the
// blob scan to be singleton, the shared queue listener needs to run
// on ALL instances so load can be scaled out
return(queueListener);
}
}
public async Task InitializeAsync()
{
// TODO (kasobol-msft) find better way
string connectionString = Environment.GetEnvironmentVariable("AzureWebJobsStorage");
if (!string.IsNullOrWhiteSpace(connectionString))
{
RandomNameResolver nameResolver = new RandomNameResolver();
Host = new HostBuilder()
.ConfigureDefaultTestHost <BlobBindingEndToEndTests>(b =>
{
b.AddAzureStorageBlobs().AddAzureStorageQueues();
b.AddAzureStorageCoreServices();
})
.ConfigureServices(services =>
{
services.AddSingleton <INameResolver>(nameResolver);
})
.Build();
JobHost = Host.GetJobHost();
var provider = Host.Services.GetService <StorageAccountProvider>();
StorageAccount = provider.GetHost();
var blobClient = StorageAccount.CreateBlobServiceClient();
BlobContainer = blobClient.GetBlobContainerClient(nameResolver.ResolveInString(ContainerName));
Assert.False(await BlobContainer.ExistsAsync());
await BlobContainer.CreateAsync();
OutputBlobContainer = blobClient.GetBlobContainerClient(nameResolver.ResolveInString(OutputContainerName));
var pageBlobContainer = blobClient.GetBlobContainerClient(nameResolver.ResolveInString(PageBlobContainerName));
Assert.False(await pageBlobContainer.ExistsAsync());
await pageBlobContainer.CreateAsync();
var hierarchicalBlobContainer = blobClient.GetBlobContainerClient(nameResolver.ResolveInString(HierarchicalBlobContainerName));
Assert.False(await hierarchicalBlobContainer.ExistsAsync());
await hierarchicalBlobContainer.CreateAsync();
var appendBlobContainer = blobClient.GetBlobContainerClient(nameResolver.ResolveInString(AppendBlobContainerName));
Assert.False(await appendBlobContainer.ExistsAsync());
await appendBlobContainer.CreateAsync();
await Host.StartAsync();
// upload some test blobs
BlockBlobClient blob = BlobContainer.GetBlockBlobClient("blob1");
await blob.UploadTextAsync(TestData);
blob = BlobContainer.GetBlockBlobClient("blob2");
await blob.UploadTextAsync(TestData);
blob = BlobContainer.GetBlockBlobClient("blob3");
await blob.UploadTextAsync(TestData);
blob = BlobContainer.GetBlockBlobClient("file1");
await blob.UploadTextAsync(TestData);
blob = BlobContainer.GetBlockBlobClient("file2");
await blob.UploadTextAsync(TestData);
blob = BlobContainer.GetBlockBlobClient("overwrite");
await blob.UploadTextAsync(TestData);
// add a couple hierarchical blob paths
blob = hierarchicalBlobContainer.GetBlockBlobClient("sub/blob1");
await blob.UploadTextAsync(TestData);
blob = hierarchicalBlobContainer.GetBlockBlobClient("sub/blob2");
await blob.UploadTextAsync(TestData);
blob = hierarchicalBlobContainer.GetBlockBlobClient("sub/sub/blob3");
await blob.UploadTextAsync(TestData);
blob = hierarchicalBlobContainer.GetBlockBlobClient("blob4");
await blob.UploadTextAsync(TestData);
byte[] bytes = new byte[512];
byte[] testBytes = Encoding.UTF8.GetBytes(TestData);
for (int i = 0; i < testBytes.Length; i++)
{
bytes[i] = testBytes[i];
}
PageBlobClient pageBlob = pageBlobContainer.GetPageBlobClient("blob1");
await pageBlob.UploadFromByteArrayAsync(bytes, 0);
pageBlob = pageBlobContainer.GetPageBlobClient("blob2");
await pageBlob.UploadFromByteArrayAsync(bytes, 0);
AppendBlobClient appendBlob = appendBlobContainer.GetAppendBlobClient("blob1");
await appendBlob.UploadTextAsync(TestData);
appendBlob = appendBlobContainer.GetAppendBlobClient("blob2");
await appendBlob.UploadTextAsync(TestData);
appendBlob = appendBlobContainer.GetAppendBlobClient("blob3");
await appendBlob.UploadTextAsync(TestData);
}
}
public BlobTests(AzuriteFixture azuriteFixture)
{
account = azuriteFixture.GetAccount();
account.CreateBlobServiceClient().GetBlobContainerClient(ContainerName).DeleteIfExists();
account.CreateQueueServiceClient().GetQueueClient(TriggerQueueName).DeleteIfExists();
}
public HostCallTests(AzuriteFixture azuriteFixture)
{
account = azuriteFixture.GetAccount();
account.CreateBlobServiceClient().GetBlobContainerClient(ContainerName).DeleteIfExists();
}
