public async Task InitializeAsync(WebJobsTestEnvironment testEnvironment)
{
RandomNameResolver nameResolver = new RandomNameResolver();
Host = new HostBuilder()
.ConfigureDefaultTestHost <BlobBindingEndToEndTests>(b =>
{
b.AddAzureStorageBlobs().AddAzureStorageQueues();
b.AddAzureStorageCoreServices();
})
.ConfigureServices(services =>
{
services.AddSingleton <INameResolver>(nameResolver);
})
.Build();
JobHost = Host.GetJobHost();
BlobServiceClient = new BlobServiceClient(testEnvironment.PrimaryStorageAccountConnectionString);
BlobContainer = BlobServiceClient.GetBlobContainerClient(nameResolver.ResolveInString(ContainerName));
Assert.False(await BlobContainer.ExistsAsync());
await BlobContainer.CreateAsync();
OutputBlobContainer = BlobServiceClient.GetBlobContainerClient(nameResolver.ResolveInString(OutputContainerName));
var pageBlobContainer = BlobServiceClient.GetBlobContainerClient(nameResolver.ResolveInString(PageBlobContainerName));
Assert.False(await pageBlobContainer.ExistsAsync());
await pageBlobContainer.CreateAsync();
var hierarchicalBlobContainer = BlobServiceClient.GetBlobContainerClient(nameResolver.ResolveInString(HierarchicalBlobContainerName));
Assert.False(await hierarchicalBlobContainer.ExistsAsync());
await hierarchicalBlobContainer.CreateAsync();
var appendBlobContainer = BlobServiceClient.GetBlobContainerClient(nameResolver.ResolveInString(AppendBlobContainerName));
Assert.False(await appendBlobContainer.ExistsAsync());
await appendBlobContainer.CreateAsync();
await Host.StartAsync();
// upload some test blobs
BlockBlobClient blob = BlobContainer.GetBlockBlobClient("blob1");
await blob.UploadTextAsync(TestData);
blob = BlobContainer.GetBlockBlobClient("blob2");
await blob.UploadTextAsync(TestData);
blob = BlobContainer.GetBlockBlobClient("blob3");
await blob.UploadTextAsync(TestData);
blob = BlobContainer.GetBlockBlobClient("file1");
await blob.UploadTextAsync(TestData);
blob = BlobContainer.GetBlockBlobClient("file2");
await blob.UploadTextAsync(TestData);
blob = BlobContainer.GetBlockBlobClient("overwrite");
await blob.UploadTextAsync(TestData);
// add a couple hierarchical blob paths
blob = hierarchicalBlobContainer.GetBlockBlobClient("sub/blob1");
await blob.UploadTextAsync(TestData);
blob = hierarchicalBlobContainer.GetBlockBlobClient("sub/blob2");
await blob.UploadTextAsync(TestData);
blob = hierarchicalBlobContainer.GetBlockBlobClient("sub/sub/blob3");
await blob.UploadTextAsync(TestData);
blob = hierarchicalBlobContainer.GetBlockBlobClient("blob4");
await blob.UploadTextAsync(TestData);
byte[] bytes = new byte[512];
byte[] testBytes = Encoding.UTF8.GetBytes(TestData);
for (int i = 0; i < testBytes.Length; i++)
{
bytes[i] = testBytes[i];
}
PageBlobClient pageBlob = pageBlobContainer.GetPageBlobClient("blob1");
await pageBlob.UploadFromByteArrayAsync(bytes, 0);
pageBlob = pageBlobContainer.GetPageBlobClient("blob2");
await pageBlob.UploadFromByteArrayAsync(bytes, 0);
AppendBlobClient appendBlob = appendBlobContainer.GetAppendBlobClient("blob1");
await appendBlob.UploadTextAsync(TestData);
appendBlob = appendBlobContainer.GetAppendBlobClient("blob2");
await appendBlob.UploadTextAsync(TestData);
appendBlob = appendBlobContainer.GetAppendBlobClient("blob3");
await appendBlob.UploadTextAsync(TestData);
}
public async Task TableFilterTest()
{
// Reinitialize the name resolver to avoid conflicts
_resolver = new RandomNameResolver();
JobHostConfiguration hostConfig = new JobHostConfiguration()
{
NameResolver = _resolver,
TypeLocator = new FakeTypeLocator(
this.GetType(),
typeof(BlobToCustomObjectBinder))
};
hostConfig.AddService <IWebJobsExceptionHandler>(new TestExceptionHandler());
// write test entities
string testTableName = _resolver.ResolveInString(TableName);
CloudTableClient tableClient = _storageAccount.CreateCloudTableClient();
CloudTable table = tableClient.GetTableReference(testTableName);
await table.CreateIfNotExistsAsync();
var operation = new TableBatchOperation();
operation.Insert(new Person
{
PartitionKey = "1",
RowKey = "1",
Name = "Lary",
Age = 20,
Location = "Seattle"
});
operation.Insert(new Person
{
PartitionKey = "1",
RowKey = "2",
Name = "Moe",
Age = 35,
Location = "Seattle"
});
operation.Insert(new Person
{
PartitionKey = "1",
RowKey = "3",
Name = "Curly",
Age = 45,
Location = "Texas"
});
operation.Insert(new Person
{
PartitionKey = "1",
RowKey = "4",
Name = "Bill",
Age = 28,
Location = "Tam O'Shanter"
});
await table.ExecuteBatchAsync(operation);
JobHost host = new JobHost(hostConfig);
var methodInfo = this.GetType().GetMethod("TableWithFilter", BindingFlags.Public | BindingFlags.Static);
var input = new Person {
Age = 25, Location = "Seattle"
};
string json = JsonConvert.SerializeObject(input);
var arguments = new { person = json };
await host.CallAsync(methodInfo, arguments);
// wait for test results to appear
await TestHelpers.Await(() => testResult != null);
JArray results = (JArray)testResult;
Assert.Equal(1, results.Count);
input = new Person {
Age = 25, Location = "Tam O'Shanter"
};
json = JsonConvert.SerializeObject(input);
arguments = new { person = json };
await host.CallAsync(methodInfo, arguments);
await TestHelpers.Await(() => testResult != null);
results = (JArray)testResult;
Assert.Equal(1, results.Count);
Assert.Equal("Bill", (string)results[0]["Name"]);
}
public async Task BadQueueMessageE2ETests()
{
// This test ensures that the host does not crash on a bad message (it previously did)
// Insert a bad message into a queue that should:
// - trigger BadMessage_CloudQueueMessage, which will put it into a second queue that will
// - trigger BadMessage_String, which should fail
// - BadMessage_String should fail repeatedly until it is moved to the poison queue
// The test will watch that poison queue to know when to complete
// Reinitialize the name resolver to avoid conflicts
_resolver = new RandomNameResolver();
JobHostConfiguration hostConfig = new JobHostConfiguration()
{
NameResolver = _resolver,
TypeLocator = new FakeTypeLocator(
this.GetType(),
typeof(BlobToCustomObjectBinder))
};
hostConfig.AddService <IWebJobsExceptionHandler>(new TestExceptionHandler());
// use a custom processor so we can grab the Id and PopReceipt
hostConfig.Queues.QueueProcessorFactory = new TestQueueProcessorFactory();
var tracer = new TestTraceWriter(System.Diagnostics.TraceLevel.Verbose);
hostConfig.Tracing.Tracers.Add(tracer);
ILoggerFactory loggerFactory = new LoggerFactory();
TestLoggerProvider loggerProvider = new TestLoggerProvider();
loggerFactory.AddProvider(loggerProvider);
hostConfig.LoggerFactory = loggerFactory;
// The jobs host is started
JobHost host = new JobHost(hostConfig);
host.Start();
// use reflection to construct a bad message:
// - use a GUID as the content, which is not a valid base64 string
// - pass 'true', to indicate that it is a base64 string
string messageContent = Guid.NewGuid().ToString();
object[] parameters = new object[] { messageContent, true };
CloudQueueMessage message = Activator.CreateInstance(typeof(CloudQueueMessage),
BindingFlags.Instance | BindingFlags.NonPublic, null, parameters, null) as CloudQueueMessage;
var queueClient = _storageAccount.CreateCloudQueueClient();
var queue = queueClient.GetQueueReference(_resolver.ResolveInString(BadMessageQueue1));
await queue.CreateIfNotExistsAsync();
await queue.ClearAsync();
// the poison queue will end up off of the second queue
var poisonQueue = queueClient.GetQueueReference(_resolver.ResolveInString(BadMessageQueue2) + "-poison");
await poisonQueue.DeleteIfExistsAsync();
await queue.AddMessageAsync(message);
CloudQueueMessage poisonMessage = null;
await TestHelpers.Await(async() =>
{
bool done = false;
if (await poisonQueue.ExistsAsync())
{
poisonMessage = await poisonQueue.GetMessageAsync();
done = poisonMessage != null;
if (done)
{
// Sleep briefly, then make sure the other message has been deleted.
// If so, trying to delete it again will throw an error.
Thread.Sleep(1000);
// The message is in the second queue
var queue2 = queueClient.GetQueueReference(_resolver.ResolveInString(BadMessageQueue2));
StorageException ex = await Assert.ThrowsAsync <StorageException>(
() => queue2.DeleteMessageAsync(_lastMessageId, _lastMessagePopReceipt));
Assert.Equal("MessageNotFound", ex.RequestInformation.ExtendedErrorInformation.ErrorCode);
}
}
return(done);
});
host.Stop();
// find the raw string to compare it to the original
Assert.NotNull(poisonMessage);
var propInfo = typeof(CloudQueueMessage).GetProperty("RawString", BindingFlags.Instance | BindingFlags.NonPublic);
string rawString = propInfo.GetValue(poisonMessage) as string;
Assert.Equal(messageContent, rawString);
// Make sure the functions were called correctly
Assert.Equal(1, _badMessage1Calls);
Assert.Equal(0, _badMessage2Calls);
// make sure the exception is being properly logged
var errors = tracer.Traces.Where(t => t.Level == System.Diagnostics.TraceLevel.Error);
Assert.True(errors.All(t => t.Exception.InnerException.InnerException is FormatException));
// Validate Logger
var loggerErrors = loggerProvider.GetAllLogMessages().Where(l => l.Level == Microsoft.Extensions.Logging.LogLevel.Error);
Assert.True(loggerErrors.All(t => t.Exception.InnerException.InnerException is FormatException));
}
public async Task InitializeAsync()
{
RandomNameResolver nameResolver = new RandomNameResolver();
Host = new HostBuilder()
.ConfigureDefaultTestHost <BlobBindingEndToEndTests>(b =>
{
b.AddAzureStorage();
RuntimeStorageWebJobsBuilderExtensions.AddAzureStorageCoreServices(b);
})
.ConfigureServices(services =>
{
services.AddSingleton <INameResolver>(nameResolver);
})
.Build();
JobHost = Host.GetJobHost();
var provider = Host.Services.GetService <StorageAccountProvider>();
StorageAccount = provider.GetHost().SdkObject;
CloudBlobClient blobClient = StorageAccount.CreateCloudBlobClient();
BlobContainer = blobClient.GetContainerReference(nameResolver.ResolveInString(ContainerName));
Assert.False(await BlobContainer.ExistsAsync());
await BlobContainer.CreateAsync();
OutputBlobContainer = blobClient.GetContainerReference(nameResolver.ResolveInString(OutputContainerName));
CloudBlobContainer pageBlobContainer = blobClient.GetContainerReference(nameResolver.ResolveInString(PageBlobContainerName));
Assert.False(await pageBlobContainer.ExistsAsync());
await pageBlobContainer.CreateAsync();
CloudBlobContainer hierarchicalBlobContainer = blobClient.GetContainerReference(nameResolver.ResolveInString(HierarchicalBlobContainerName));
Assert.False(await hierarchicalBlobContainer.ExistsAsync());
await hierarchicalBlobContainer.CreateAsync();
CloudBlobContainer appendBlobContainer = blobClient.GetContainerReference(nameResolver.ResolveInString(AppendBlobContainerName));
Assert.False(await appendBlobContainer.ExistsAsync());
await appendBlobContainer.CreateAsync();
await Host.StartAsync();
// upload some test blobs
CloudBlockBlob blob = BlobContainer.GetBlockBlobReference("blob1");
await blob.UploadTextAsync(TestData);
blob = BlobContainer.GetBlockBlobReference("blob2");
await blob.UploadTextAsync(TestData);
blob = BlobContainer.GetBlockBlobReference("blob3");
await blob.UploadTextAsync(TestData);
blob = BlobContainer.GetBlockBlobReference("file1");
await blob.UploadTextAsync(TestData);
blob = BlobContainer.GetBlockBlobReference("file2");
await blob.UploadTextAsync(TestData);
blob = BlobContainer.GetBlockBlobReference("overwrite");
await blob.UploadTextAsync(TestData);
// add a couple hierarchical blob paths
blob = hierarchicalBlobContainer.GetBlockBlobReference("sub/blob1");
await blob.UploadTextAsync(TestData);
blob = hierarchicalBlobContainer.GetBlockBlobReference("sub/blob2");
await blob.UploadTextAsync(TestData);
blob = hierarchicalBlobContainer.GetBlockBlobReference("sub/sub/blob3");
await blob.UploadTextAsync(TestData);
blob = hierarchicalBlobContainer.GetBlockBlobReference("blob4");
await blob.UploadTextAsync(TestData);
byte[] bytes = new byte[512];
byte[] testBytes = Encoding.UTF8.GetBytes(TestData);
for (int i = 0; i < testBytes.Length; i++)
{
bytes[i] = testBytes[i];
}
CloudPageBlob pageBlob = pageBlobContainer.GetPageBlobReference("blob1");
await pageBlob.UploadFromByteArrayAsync(bytes, 0, bytes.Length);
pageBlob = pageBlobContainer.GetPageBlobReference("blob2");
await pageBlob.UploadFromByteArrayAsync(bytes, 0, bytes.Length);
CloudAppendBlob appendBlob = appendBlobContainer.GetAppendBlobReference("blob1");
await appendBlob.UploadTextAsync(TestData);
appendBlob = appendBlobContainer.GetAppendBlobReference("blob2");
await appendBlob.UploadTextAsync(TestData);
appendBlob = appendBlobContainer.GetAppendBlobReference("blob3");
await appendBlob.UploadTextAsync(TestData);
}
public async Task BadQueueMessageE2ETests()
{
// This test ensures that the host does not crash on a bad message (it previously did)
// Insert a bad message into a queue that should:
// - trigger BadMessage_CloudQueueMessage, which will put it into a second queue that will
// - trigger BadMessage_String, which should fail
// - BadMessage_String should fail repeatedly until it is moved to the poison queue
// The test will watch that poison queue to know when to complete
// Reinitialize the name resolver to avoid conflicts
_resolver = new RandomNameResolver();
IHost host = new HostBuilder()
.ConfigureDefaultTestHost <AzureStorageEndToEndTests>(b =>
{
b.AddAzureStorageBlobs().AddAzureStorageQueues();
})
.ConfigureServices(services =>
{
// use a custom processor so we can grab the Id and PopReceipt
services.AddSingleton <IQueueProcessorFactory>(new TestQueueProcessorFactory());
services.AddSingleton <INameResolver>(_resolver);
})
.Build();
TestLoggerProvider loggerProvider = host.GetTestLoggerProvider();
// The jobs host is started
host.Start();
// Construct a bad message:
// - use a GUID as the content, which is not a valid base64 string
// - pass 'true', to indicate that it is a base64 string
string messageContent = Guid.NewGuid().ToString();
// var message = new CloudQueueMessage(messageContent, true); // TODO (kasobol-msft) check this base64 thing
var queue = _queueServiceClient.GetQueueClient(_resolver.ResolveInString(BadMessageQueue1));
await queue.CreateIfNotExistsAsync();
await queue.ClearMessagesAsync();
// the poison queue will end up off of the second queue
var poisonQueue = _queueServiceClient.GetQueueClient(_resolver.ResolveInString(BadMessageQueue2) + "-poison");
await poisonQueue.DeleteIfExistsAsync();
await queue.SendMessageAsync(messageContent);
QueueMessage poisonMessage = null;
await TestHelpers.Await(async() =>
{
bool done = false;
if (await poisonQueue.ExistsAsync())
{
poisonMessage = (await poisonQueue.ReceiveMessagesAsync(1)).Value.FirstOrDefault();
done = poisonMessage != null;
if (done)
{
// Sleep briefly, then make sure the other message has been deleted.
// If so, trying to delete it again will throw an error.
Thread.Sleep(1000);
// The message is in the second queue
var queue2 = _queueServiceClient.GetQueueClient(_resolver.ResolveInString(BadMessageQueue2));
RequestFailedException ex = Assert.ThrowsAsync <RequestFailedException>(
() => queue2.DeleteMessageAsync(_lastMessageId, _lastMessagePopReceipt));
Assert.AreEqual("MessageNotFound", ex.ErrorCode);
}
}
var logs = loggerProvider.GetAllLogMessages();
return(done);
});
await host.StopAsync();
// find the raw string to compare it to the original
Assert.NotNull(poisonMessage);
Assert.AreEqual(messageContent, poisonMessage.MessageText);
// Make sure the functions were called correctly
Assert.AreEqual(1, _badMessage1Calls);
Assert.AreEqual(0, _badMessage2Calls);
// Validate Logger
var loggerErrors = loggerProvider.GetAllLogMessages().Where(l => l.Level == Microsoft.Extensions.Logging.LogLevel.Error);
Assert.True(loggerErrors.All(t => t.Exception.InnerException.InnerException is FormatException));
}
public async Task BadQueueMessageE2ETests()
{
// This test ensures that the host does not crash on a bad message (it previously did)
// Insert a bad message into a queue that should:
// - trigger BadMessage_CloudQueueMessage, which will put it into a second queue that will
// - trigger BadMessage_String, which should fail
// - BadMessage_String should fail repeatedly until it is moved to the poison queue
// The test will watch that poison queue to know when to complete
// Reinitialize the name resolver to avoid conflicts
_resolver = new RandomNameResolver();
JobHostConfiguration hostConfig = new JobHostConfiguration()
{
NameResolver = _resolver,
TypeLocator = new FakeTypeLocator(
this.GetType(),
typeof(BlobToCustomObjectBinder))
};
var tracer = new TestTraceWriter(TraceLevel.Verbose);
hostConfig.Tracing.Tracers.Add(tracer);
// The jobs host is started
JobHost host = new JobHost(hostConfig);
host.Start();
// use reflection to construct a bad message:
// - use a GUID as the content, which is not a valid base64 string
// - pass 'true', to indicate that it is a base64 string
string messageContent = Guid.NewGuid().ToString();
object[] parameters = new object[] { messageContent, true };
CloudQueueMessage message = Activator.CreateInstance(typeof(CloudQueueMessage),
BindingFlags.Instance | BindingFlags.NonPublic, null, parameters, null) as CloudQueueMessage;
var queueClient = _storageAccount.CreateCloudQueueClient();
var queue = queueClient.GetQueueReference(_resolver.ResolveInString(BadMessageQueue1));
await queue.CreateIfNotExistsAsync();
await queue.ClearAsync();
// the poison queue will end up off of the second queue
var poisonQueue = queueClient.GetQueueReference(_resolver.ResolveInString(BadMessageQueue2) + "-poison");
await poisonQueue.DeleteIfExistsAsync();
await queue.AddMessageAsync(message);
CloudQueueMessage poisonMessage = null;
await TestHelpers.Await(() =>
{
bool done = false;
if (poisonQueue.Exists())
{
poisonMessage = poisonQueue.GetMessage();
done = poisonMessage != null;
}
return(done);
});
host.Stop();
// find the raw string to compare it to the original
Assert.NotNull(poisonMessage);
var propInfo = typeof(CloudQueueMessage).GetProperty("RawString", BindingFlags.Instance | BindingFlags.NonPublic);
string rawString = propInfo.GetValue(poisonMessage) as string;
Assert.Equal(messageContent, rawString);
// Make sure the functions were called correctly
Assert.Equal(1, _badMessage1Calls);
Assert.Equal(0, _badMessage2Calls);
// make sure the exception is being properly logged
var errors = tracer.Traces.Where(t => t.Level == TraceLevel.Error);
Assert.True(errors.All(t => t.Exception.InnerException.InnerException is FormatException));
}
public TestFixture()
{
RandomNameResolver nameResolver = new RandomNameResolver();
JobHostConfiguration hostConfiguration = new JobHostConfiguration()
{
NameResolver = nameResolver,
TypeLocator = new FakeTypeLocator(typeof(MultipleStorageAccountsEndToEndTests)),
};
Config = hostConfiguration;
Account1 = CloudStorageAccount.Parse(hostConfiguration.StorageConnectionString);
string secondaryConnectionString = AmbientConnectionStringProvider.Instance.GetConnectionString(Secondary);
Account2 = CloudStorageAccount.Parse(secondaryConnectionString);
CleanContainers();
CloudBlobClient blobClient1 = Account1.CreateCloudBlobClient();
string inputName = nameResolver.ResolveInString(Input);
CloudBlobContainer inputContainer1 = blobClient1.GetContainerReference(inputName);
inputContainer1.Create();
string outputName = nameResolver.ResolveWholeString(Output);
OutputContainer1 = blobClient1.GetContainerReference(outputName);
OutputContainer1.CreateIfNotExists();
CloudBlobClient blobClient2 = Account2.CreateCloudBlobClient();
CloudBlobContainer inputContainer2 = blobClient2.GetContainerReference(inputName);
inputContainer2.Create();
OutputContainer2 = blobClient2.GetContainerReference(outputName);
OutputContainer2.CreateIfNotExists();
CloudQueueClient queueClient1 = Account1.CreateCloudQueueClient();
CloudQueue inputQueue1 = queueClient1.GetQueueReference(inputName);
inputQueue1.CreateIfNotExists();
OutputQueue1 = queueClient1.GetQueueReference(outputName);
OutputQueue1.CreateIfNotExists();
CloudQueueClient queueClient2 = Account2.CreateCloudQueueClient();
CloudQueue inputQueue2 = queueClient2.GetQueueReference(inputName);
inputQueue2.CreateIfNotExists();
OutputQueue2 = queueClient2.GetQueueReference(outputName);
OutputQueue2.CreateIfNotExists();
CloudTableClient tableClient1 = Account1.CreateCloudTableClient();
string outputTableName = nameResolver.ResolveWholeString(OutputTableName);
OutputTable1 = tableClient1.GetTableReference(outputTableName);
OutputTable2 = Account2.CreateCloudTableClient().GetTableReference(outputTableName);
// upload some test blobs to the input containers of both storage accounts
CloudBlockBlob blob = inputContainer1.GetBlockBlobReference("blob1");
blob.UploadText(TestData);
blob = inputContainer2.GetBlockBlobReference("blob2");
blob.UploadText(TestData);
// upload some test queue messages to the input queues of both storage accounts
inputQueue1.AddMessage(new CloudQueueMessage(TestData));
inputQueue2.AddMessage(new CloudQueueMessage(TestData));
Host = new JobHost(hostConfiguration);
Host.Start();
}
public async Task BadQueueMessageE2ETests()
{
// This test ensures that the host does not crash on a bad message (it previously did)
// Insert a bad message into a queue that should:
// - trigger BadMessage_String, which should fail
// - BadMessage_String should be transfered to poison queue.
// The test will watch that poison queue to know when to complete
// Reinitialize the name resolver to avoid conflicts
_resolver = new RandomNameResolver();
IHost host = new HostBuilder()
.ConfigureDefaultTestHost <AzureStorageEndToEndTests>(b =>
{
b.AddAzureStorageBlobs().AddAzureStorageQueues();
})
.ConfigureServices(services =>
{
// use a custom processor so we can grab the Id and PopReceipt
services.AddSingleton <IQueueProcessorFactory>(new TestQueueProcessorFactory());
services.AddSingleton <INameResolver>(_resolver);
})
.Build();
TestLoggerProvider loggerProvider = host.GetTestLoggerProvider();
// The jobs host is started
host.Start();
// Construct a bad message:
// - use a GUID as the content, which is not a valid base64 string
// - pass 'true', to indicate that it is a base64 string
string messageContent = Guid.NewGuid().ToString();
var queue = _queueServiceClientWithoutEncoding.GetQueueClient(_resolver.ResolveInString(BadMessageQueue));
await queue.CreateIfNotExistsAsync();
await queue.ClearMessagesAsync();
var poisonQueue = _queueServiceClientWithoutEncoding.GetQueueClient(_resolver.ResolveInString(BadMessageQueue) + "-poison");
await poisonQueue.DeleteIfExistsAsync();
await queue.SendMessageAsync(messageContent);
QueueMessage poisonMessage = null;
await TestHelpers.Await(async() =>
{
bool done = false;
if (await poisonQueue.ExistsAsync())
{
poisonMessage = await poisonQueue.ReceiveMessageAsync();
done = poisonMessage != null;
}
var logs = loggerProvider.GetAllLogMessages();
return(done);
});
await host.StopAsync();
// find the raw string to compare it to the original
Assert.NotNull(poisonMessage);
Assert.AreEqual(messageContent, poisonMessage.MessageText);
// Make sure the functions were called correctly
Assert.AreEqual(0, _badMessageCalls);
// Validate Logger
var loggerErrors = loggerProvider.GetAllLogMessages().Where(l => l.Level == Microsoft.Extensions.Logging.LogLevel.Error);
Assert.True(loggerErrors.All(t => t.Exception.InnerException.InnerException is FormatException));
}
public async Task DynamicConcurrency_Blobs()
{
// Reinitialize the name resolver to avoid conflicts
_resolver = new RandomNameResolver();
DynamicConcurrencyTestJob.InvocationCount = 0;
IHost host = new HostBuilder()
.ConfigureDefaultTestHost <DynamicConcurrencyTestJob>(b =>
{
b.AddAzureStorageBlobs();
b.Services.AddOptions <ConcurrencyOptions>().Configure(options =>
{
options.DynamicConcurrencyEnabled = true;
});
b.Services.AddOptions <QueuesOptions>().Configure(options =>
{
options.MaxPollingInterval = TimeSpan.FromSeconds(1);
});
})
.ConfigureServices(services =>
{
services.AddSingleton <INameResolver>(_resolver);
})
.ConfigureLogging((context, b) =>
{
b.SetMinimumLevel(LogLevel.Debug);
})
.Build();
string sharedListenerId = "SharedBlobQueueListener";
var concurrencyManager = host.Services.GetServices <ConcurrencyManager>().SingleOrDefault();
var concurrencyStatus = concurrencyManager.GetStatus(sharedListenerId);
Assert.AreEqual(1, concurrencyStatus.CurrentConcurrency);
// write some blobs
int numBlobs = 50;
string blobContainerName = _resolver.ResolveInString(DynamicConcurrencyBlobContainerName);
await WriteBlobs(blobContainerName, numBlobs);
// start the host
await host.StartAsync();
// wait for all messages to be processed
await TestHelpers.Await(() =>
{
return(DynamicConcurrencyTestJob.InvocationCount >= numBlobs);
});
await host.StopAsync();
// ensure we've dynamically increased concurrency
concurrencyStatus = concurrencyManager.GetStatus("SharedBlobQueueListener");
Assert.GreaterOrEqual(concurrencyStatus.CurrentConcurrency, 3);
// check a few of the concurrency logs
var concurrencyLogs = host.GetTestLoggerProvider().GetAllLogMessages().Where(p => p.Category == LogCategories.Concurrency).Select(p => p.FormattedMessage).ToList();
int concurrencyIncreaseLogCount = concurrencyLogs.Count(p => p.Contains($"{sharedListenerId} Increasing concurrency"));
Assert.GreaterOrEqual(concurrencyIncreaseLogCount, 1);
}
private async Task Initialize()
{
RandomNameResolver nameResolver = new RandomNameResolver();
JobHostConfiguration hostConfiguration = new JobHostConfiguration()
{
NameResolver = nameResolver,
TypeLocator = new FakeTypeLocator(typeof(BlobBindingEndToEndTests)),
};
hostConfiguration.AddService <IWebJobsExceptionHandler>(new TestExceptionHandler());
Config = hostConfiguration;
StorageAccount = CloudStorageAccount.Parse(hostConfiguration.StorageConnectionString);
CloudBlobClient blobClient = StorageAccount.CreateCloudBlobClient();
BlobContainer = blobClient.GetContainerReference(nameResolver.ResolveInString(ContainerName));
Assert.False(await BlobContainer.ExistsAsync());
await BlobContainer.CreateAsync();
OutputBlobContainer = blobClient.GetContainerReference(nameResolver.ResolveInString(OutputContainerName));
CloudBlobContainer pageBlobContainer = blobClient.GetContainerReference(nameResolver.ResolveInString(PageBlobContainerName));
Assert.False(await pageBlobContainer.ExistsAsync());
await pageBlobContainer.CreateAsync();
CloudBlobContainer hierarchicalBlobContainer = blobClient.GetContainerReference(nameResolver.ResolveInString(HierarchicalBlobContainerName));
Assert.False(await hierarchicalBlobContainer.ExistsAsync());
await hierarchicalBlobContainer.CreateAsync();
CloudBlobContainer appendBlobContainer = blobClient.GetContainerReference(nameResolver.ResolveInString(AppendBlobContainerName));
Assert.False(await appendBlobContainer.ExistsAsync());
await appendBlobContainer.CreateAsync();
Host = new JobHost(hostConfiguration);
Host.Start();
// upload some test blobs
CloudBlockBlob blob = BlobContainer.GetBlockBlobReference("blob1");
await blob.UploadTextAsync(TestData);
blob = BlobContainer.GetBlockBlobReference("blob2");
await blob.UploadTextAsync(TestData);
blob = BlobContainer.GetBlockBlobReference("blob3");
await blob.UploadTextAsync(TestData);
blob = BlobContainer.GetBlockBlobReference("file1");
await blob.UploadTextAsync(TestData);
blob = BlobContainer.GetBlockBlobReference("file2");
await blob.UploadTextAsync(TestData);
blob = BlobContainer.GetBlockBlobReference("overwrite");
await blob.UploadTextAsync(TestData);
// add a couple hierarchical blob paths
blob = hierarchicalBlobContainer.GetBlockBlobReference("sub/blob1");
await blob.UploadTextAsync(TestData);
blob = hierarchicalBlobContainer.GetBlockBlobReference("sub/blob2");
await blob.UploadTextAsync(TestData);
blob = hierarchicalBlobContainer.GetBlockBlobReference("sub/sub/blob3");
await blob.UploadTextAsync(TestData);
blob = hierarchicalBlobContainer.GetBlockBlobReference("blob4");
await blob.UploadTextAsync(TestData);
byte[] bytes = new byte[512];
byte[] testBytes = Encoding.UTF8.GetBytes(TestData);
for (int i = 0; i < testBytes.Length; i++)
{
bytes[i] = testBytes[i];
}
CloudPageBlob pageBlob = pageBlobContainer.GetPageBlobReference("blob1");
await pageBlob.UploadFromByteArrayAsync(bytes, 0, bytes.Length);
pageBlob = pageBlobContainer.GetPageBlobReference("blob2");
await pageBlob.UploadFromByteArrayAsync(bytes, 0, bytes.Length);
CloudAppendBlob appendBlob = appendBlobContainer.GetAppendBlobReference("blob1");
await appendBlob.UploadTextAsync(TestData);
appendBlob = appendBlobContainer.GetAppendBlobReference("blob2");
await appendBlob.UploadTextAsync(TestData);
appendBlob = appendBlobContainer.GetAppendBlobReference("blob3");
await appendBlob.UploadTextAsync(TestData);
}