public async Task ExecuteAsync_UpdatesScanInfoManager()
{
int testScanBlobLimitPerPoll = 6;
string firstContainerName = Guid.NewGuid().ToString();
string secondContainerName = Guid.NewGuid().ToString();
IStorageAccount account = CreateFakeStorageAccount();
IStorageBlobContainer firstContainer = account.CreateBlobClient().GetContainerReference(firstContainerName);
IStorageBlobContainer secondContainer = account.CreateBlobClient().GetContainerReference(secondContainerName);
TestBlobScanInfoManager testScanInfoManager = new TestBlobScanInfoManager();
string accountName = account.Credentials.AccountName;
testScanInfoManager.SetScanInfo(accountName, firstContainerName, DateTime.MinValue);
testScanInfoManager.SetScanInfo(accountName, secondContainerName, DateTime.MinValue);
IBlobListenerStrategy product = new ScanBlobScanLogHybridPollingStrategy(testScanInfoManager);
LambdaBlobTriggerExecutor executor = new LambdaBlobTriggerExecutor();
typeof(ScanBlobScanLogHybridPollingStrategy)
.GetField("_scanBlobLimitPerPoll", BindingFlags.Instance | BindingFlags.NonPublic)
.SetValue(product, testScanBlobLimitPerPoll);
await product.RegisterAsync(firstContainer, executor, CancellationToken.None);
await product.RegisterAsync(secondContainer, executor, CancellationToken.None);
var firstExpectedNames = new List <string>();
for (int i = 0; i < 3; i++)
{
firstExpectedNames.Add(CreateAblobAndUploadToContainer(firstContainer));
}
RunExecuteWithMultiPollingInterval(firstExpectedNames, product, executor, testScanBlobLimitPerPoll / 2);
// only expect the first container to have updated its scanInfo
Assert.Equal(1, testScanInfoManager.UpdateCounts[accountName][firstContainerName]);
int count;
testScanInfoManager.UpdateCounts[accountName].TryGetValue(secondContainerName, out count);
Assert.Equal(0, count);
await Task.Delay(10);
var secondExpectedNames = new List <string>();
for (int i = 0; i < 7; i++)
{
secondExpectedNames.Add(CreateAblobAndUploadToContainer(secondContainer));
}
RunExecuteWithMultiPollingInterval(secondExpectedNames, product, executor, testScanBlobLimitPerPoll / 2);
// this time, only expect the second container to have updated its scanInfo
Assert.Equal(1, testScanInfoManager.UpdateCounts[accountName][firstContainerName]);
Assert.Equal(1, testScanInfoManager.UpdateCounts[accountName][secondContainerName]);
}
public async Task <IListener> CreateAsync(IFunctionExecutor executor, CancellationToken cancellationToken)
{
SharedQueueWatcher sharedQueueWatcher = _sharedContextProvider.GetOrCreate <SharedQueueWatcher>(
new SharedQueueWatcherFactory(_messageEnqueuedWatcherSetter));
SharedBlobListener sharedBlobListener = _sharedContextProvider.GetOrCreate <SharedBlobListener>(
new SharedBlobListenerFactory(_account, _backgroundExceptionDispatcher, _blobWrittenWatcherSetter));
// Note that these clients are intentionally for the storage account rather than for the dashboard account.
// We use the storage, not dashboard, account for the blob receipt container and blob trigger queues.
IStorageQueueClient queueClient = _account.CreateQueueClient();
IStorageBlobClient blobClient = _account.CreateBlobClient();
string hostId = await _hostIdProvider.GetHostIdAsync(cancellationToken);
string hostBlobTriggerQueueName = HostQueueNames.GetHostBlobTriggerQueueName(hostId);
IStorageQueue hostBlobTriggerQueue = queueClient.GetQueueReference(hostBlobTriggerQueueName);
IListener blobDiscoveryToQueueMessageListener = await CreateBlobDiscoveryToQueueMessageListenerAsync(
hostId, sharedBlobListener, blobClient, hostBlobTriggerQueue, sharedQueueWatcher, cancellationToken);
IListener queueMessageToTriggerExecutionListener = CreateQueueMessageToTriggerExecutionListener(executor,
_sharedContextProvider, sharedQueueWatcher, queueClient, hostBlobTriggerQueue, blobClient,
sharedBlobListener.BlobWritterWatcher);
IListener compositeListener = new CompositeListener(
blobDiscoveryToQueueMessageListener,
queueMessageToTriggerExecutionListener);
return(compositeListener);
}
public void TestBlobListenerWithContainerBiggerThanThreshold()
{
int testScanBlobLimitPerPoll = 1;
string containerName = Path.GetRandomFileName();
IStorageAccount account = CreateFakeStorageAccount();
IStorageBlobContainer container = account.CreateBlobClient().GetContainerReference(containerName);
IBlobListenerStrategy product = new ScanBlobScanLogHybridPollingStrategy(new TestBlobScanInfoManager());
LambdaBlobTriggerExecutor executor = new LambdaBlobTriggerExecutor();
typeof(ScanBlobScanLogHybridPollingStrategy)
.GetField("_scanBlobLimitPerPoll", BindingFlags.Instance | BindingFlags.NonPublic)
.SetValue(product, testScanBlobLimitPerPoll);
product.Register(container, executor);
product.Start();
// populate with 5 blobs
List <string> expectedNames = new List <string>();
for (int i = 0; i < 5; i++)
{
expectedNames.Add(CreateAblobAndUploadToContainer(container));
}
RunExecuteWithMultiPollingInterval(expectedNames, product, executor, testScanBlobLimitPerPoll);
// Now run again; shouldn't show up.
RunExecuterWithExpectedBlobs(new List <string>(), product, executor);
}
private async Task EnsureLoggersAsync(CancellationToken cancellationToken)
{
if (_loggersSet)
{
return;
}
IStorageAccount dashboardAccount =
await _storageAccountProvider.GetDashboardAccountAsync(cancellationToken);
if (dashboardAccount != null)
{
// Create logging against a live Azure account.
IStorageBlobClient dashboardBlobClient = dashboardAccount.CreateBlobClient();
IPersistentQueueWriter <PersistentQueueMessage> queueWriter =
new PersistentQueueWriter <PersistentQueueMessage>(dashboardBlobClient);
PersistentQueueLogger queueLogger = new PersistentQueueLogger(queueWriter);
_hostInstanceLogger = queueLogger;
_functionInstanceLogger = new CompositeFunctionInstanceLogger(queueLogger,
new ConsoleFunctionInstanceLogger());
_functionOutputLogger = new BlobFunctionOutputLogger(dashboardBlobClient);
}
else
{
// No auxillary logging. Logging interfaces are nops or in-memory.
_hostInstanceLogger = new NullHostInstanceLogger();
_functionInstanceLogger = new ConsoleFunctionInstanceLogger();
_functionOutputLogger = new ConsoleFunctionOutputLogger();
}
_loggersSet = true;
}
public async Task <IBinding> TryCreateAsync(BindingProviderContext context)
{
ParameterInfo parameter = context.Parameter;
BlobAttribute blob = parameter.GetCustomAttribute <BlobAttribute>(inherit: false);
if (blob == null)
{
return(null);
}
string resolvedCombinedPath = Resolve(blob.BlobPath);
IBindableBlobPath path = BindableBlobPath.Create(resolvedCombinedPath);
path.ValidateContractCompatibility(context.BindingDataContract);
IBlobArgumentBinding argumentBinding = _provider.TryCreate(parameter, blob.Access);
if (argumentBinding == null)
{
throw new InvalidOperationException("Can't bind Blob to type '" + parameter.ParameterType + "'.");
}
IStorageAccount account = await _accountProvider.GetStorageAccountAsync(context.CancellationToken);
IStorageBlobClient client = account.CreateBlobClient();
IBinding binding = new BlobBinding(parameter.Name, argumentBinding, client, path);
return(binding);
}
public async Task RegisterAsync_InitializesWithScanInfoManager()
{
string containerName = Guid.NewGuid().ToString();
IStorageAccount account = CreateFakeStorageAccount();
IStorageBlobContainer container = account.CreateBlobClient().GetContainerReference(containerName);
TestBlobScanInfoManager scanInfoManager = new TestBlobScanInfoManager();
IBlobListenerStrategy product = new ScanBlobScanLogHybridPollingStrategy(scanInfoManager);
LambdaBlobTriggerExecutor executor = new LambdaBlobTriggerExecutor();
// Create a few blobs.
for (int i = 0; i < 5; i++)
{
CreateAblobAndUploadToContainer(container);
}
await scanInfoManager.UpdateLatestScanAsync(account.Credentials.AccountName, containerName, DateTime.UtcNow);
await product.RegisterAsync(container, executor, CancellationToken.None);
// delay slightly so we guarantee a later timestamp
await Task.Delay(10);
var expectedNames = new List <string>();
expectedNames.Add(CreateAblobAndUploadToContainer(container));
RunExecuterWithExpectedBlobs(expectedNames, product, executor);
}
public async Task ExecuteAsync_UpdatesScanInfo_WithEarliestFailure()
{
int testScanBlobLimitPerPoll = 6;
string containerName = Guid.NewGuid().ToString();
// we'll introduce multiple errors to make sure we take the earliest timestamp
DateTime earliestErrorTime = DateTime.UtcNow;
var timeMap = new Dictionary <string, DateTimeOffset>();
IStorageAccount account = CreateFakeStorageAccount();
IStorageBlobContainer container = new SkewableFakeStorageBlobContainer(new MemoryBlobStore(), containerName,
account.CreateBlobClient(), blobs =>
{
// Set a blob with "throw" to a specific date and time. Make sure the error blob
// is earlier than the others.
foreach (IStorageBlob blob in blobs.Results)
{
((FakeStorageBlobProperties)blob.Properties).LastModified = timeMap[blob.Name];
}
});
TestBlobScanInfoManager testScanInfoManager = new TestBlobScanInfoManager();
string accountName = account.Credentials.AccountName;
testScanInfoManager.SetScanInfo(accountName, containerName, DateTime.MinValue);
IBlobListenerStrategy product = new ScanBlobScanLogHybridPollingStrategy(testScanInfoManager);
LambdaBlobTriggerExecutor executor = new LambdaBlobTriggerExecutor();
typeof(ScanBlobScanLogHybridPollingStrategy)
.GetField("_scanBlobLimitPerPoll", BindingFlags.Instance | BindingFlags.NonPublic)
.SetValue(product, testScanBlobLimitPerPoll);
await product.RegisterAsync(container, executor, CancellationToken.None);
// Induce a failure to make sure the timestamp is earlier than the failure.
var expectedNames = new List <string>();
for (int i = 0; i < 7; i++)
{
string name;
if (i % 3 == 0)
{
name = CreateAblobAndUploadToContainer(container, "throw");
timeMap[name] = earliestErrorTime.AddMinutes(i);
}
else
{
name = CreateAblobAndUploadToContainer(container, "test");
timeMap[name] = earliestErrorTime.AddMinutes(10);
}
expectedNames.Add(name);
}
RunExecuteWithMultiPollingInterval(expectedNames, product, executor, testScanBlobLimitPerPoll);
DateTime?storedTime = await testScanInfoManager.LoadLatestScanAsync(accountName, containerName);
Assert.True(storedTime < earliestErrorTime);
Assert.Equal(1, testScanInfoManager.UpdateCounts[accountName][containerName]);
}
private static IStorageBlob CreateBlobReference(string containerName, string blobName)
{
IStorageAccount account = CreateAccount();
IStorageBlobClient client = account.CreateBlobClient();
IStorageBlobContainer container = client.GetContainerReference(containerName);
return(container.GetBlockBlobReference(blobName));
}
private static IStorageBlobContainer CreateContainer(IStorageAccount account, string containerName)
{
IStorageBlobClient client = account.CreateBlobClient();
IStorageBlobContainer container = client.GetContainerReference(containerName);
container.CreateIfNotExists();
return(container);
}
public void TestBlobListenerWithMultipleContainers()
{
int testScanBlobLimitPerPoll = 6, containerCount = 2;
string firstContainerName = Path.GetRandomFileName();
string secondContainerName = Path.GetRandomFileName();
IStorageAccount account = CreateFakeStorageAccount();
IStorageBlobContainer firstContainer = account.CreateBlobClient().GetContainerReference(firstContainerName);
IStorageBlobContainer secondContainer = account.CreateBlobClient().GetContainerReference(secondContainerName);
IBlobListenerStrategy product = new ScanBlobScanLogHybridPollingStrategy(new TestBlobScanInfoManager());
LambdaBlobTriggerExecutor executor = new LambdaBlobTriggerExecutor();
typeof(ScanBlobScanLogHybridPollingStrategy)
.GetField("_scanBlobLimitPerPoll", BindingFlags.Instance | BindingFlags.NonPublic)
.SetValue(product, testScanBlobLimitPerPoll);
product.Register(firstContainer, executor);
product.Register(secondContainer, executor);
product.Start();
// populate first container with 5 blobs > page size and second with 2 blobs < page size
// page size is going to be testScanBlobLimitPerPoll / number of container 6/2 = 3
List <string> firstContainerExpectedNames = new List <string>();
for (int i = 0; i < 5; i++)
{
firstContainerExpectedNames.Add(CreateAblobAndUploadToContainer(firstContainer));
}
RunExecuteWithMultiPollingInterval(firstContainerExpectedNames, product, executor, testScanBlobLimitPerPoll / containerCount);
Thread.Sleep(10);
List <string> secondContainerExpectedNames = new List <string>();
for (int i = 0; i < 2; i++)
{
secondContainerExpectedNames.Add(CreateAblobAndUploadToContainer(secondContainer));
}
RunExecuteWithMultiPollingInterval(secondContainerExpectedNames, product, executor, testScanBlobLimitPerPoll / containerCount);
// Now run again; shouldn't show up.
RunExecuterWithExpectedBlobs(new List <string>(), product, executor);
}
// Test that the credentials are valid and classify the account.Type as one of StorageAccountTypes
private static async Task ValidateCredentialsAsyncCore(IStorageAccount account, CancellationToken cancellationToken)
{
// Verify the credentials are correct.
// Have to actually ping a storage operation.
IStorageBlobClient client = account.CreateBlobClient();
try
{
// This can hang for a long time if the account name is wrong.
// If will fail fast if the password is incorrect.
await client.GetServicePropertiesAsync(cancellationToken);
}
catch (OperationCanceledException)
{
throw;
}
catch (Exception e)
{
var storageException = e as StorageException;
var isDevStoreAccount = GetIsDevStoreAccountFromCloudStorageAccount(account.SdkObject);
if (storageException?.RequestInformation?.HttpStatusCode == 400 &&
storageException?.RequestInformation?.ExtendedErrorInformation?.ErrorCode == "InvalidQueryParameterValue")
{
// Premium storage accounts do not support the GetServicePropertiesAsync call, and respond with a 400 'InvalidQueryParameterValue'.
// If we see this error response classify the account as a premium account
account.Type = StorageAccountType.Premium;
return;
}
else if (isDevStoreAccount)
{
// If using the storage emulator, it might not be running
throw new InvalidOperationException(Constants.CheckAzureStorageEmulatorMessage, e);
}
else
{
// If not a recognized error, the credentials are invalid
string message = String.Format(CultureInfo.CurrentCulture,
"Invalid storage account '{0}'. Please make sure your credentials are correct.",
account.Credentials.AccountName);
throw new InvalidOperationException(message, e);
}
}
IStorageQueueClient queueClient = account.CreateQueueClient();
IStorageQueue queue = queueClient.GetQueueReference("name");
try
{
await queue.ExistsAsync(cancellationToken);
}
catch (StorageException exception) when(IsBlobOnlyStorageException(exception))
{
account.Type = StorageAccountType.BlobOnly;
}
}
private async Task <IStorageBlobClient> GetClientAsync(
BlobAttribute blobAttribute,
CancellationToken cancellationToken)
{
IStorageAccount account = await _accountProvider.GetStorageAccountAsync(blobAttribute, cancellationToken, _nameResolver);
IStorageBlobClient client = account.CreateBlobClient();
return(client);
}
private async Task ValidateCredentialsAsyncCore(IStorageAccount account,
bool isPrimaryAccount, CancellationToken cancellationToken)
{
// Verify the credentials are correct.
// Have to actually ping a storage operation.
IStorageBlobClient client = account.CreateBlobClient();
try
{
// This can hang for a long time if the account name is wrong.
// If will fail fast if the password is incorrect.
await client.GetServicePropertiesAsync(cancellationToken);
}
catch (OperationCanceledException)
{
throw;
}
catch
{
string message = String.Format(CultureInfo.CurrentCulture,
"Invalid storage account '{0}'. Please make sure your credentials are correct.",
account.Credentials.AccountName);
throw new InvalidOperationException(message);
}
if (isPrimaryAccount)
{
// Primary storage accounts require Queues
IStorageQueueClient queueClient = account.CreateQueueClient();
IStorageQueue queue = queueClient.GetQueueReference("name");
try
{
await queue.ExistsAsync(cancellationToken);
_primaryCredentials = account.Credentials;
}
catch (OperationCanceledException)
{
throw;
}
catch (StorageException exception)
{
WebException webException = exception.GetBaseException() as WebException;
if (webException != null && webException.Status == WebExceptionStatus.NameResolutionFailure)
{
string message = String.Format(CultureInfo.CurrentCulture,
"Invalid storage account '{0}'. Primary storage accounts must be general "
+ "purpose accounts and not restricted blob storage accounts.", account.Credentials.AccountName);
throw new InvalidOperationException(message);
}
throw;
}
}
}
protected override IStorageBlobContainer GetContainer(string accountName)
{
// Get the container via a full connection string
Task <IStorageAccount> task = _accountProvider.GetStorageAccountAsync(accountName, CancellationToken.None);
IStorageAccount storageAccount = task.Result;
// singleton requires block blobs, cannot be premium
storageAccount.AssertTypeOneOf(StorageAccountType.GeneralPurpose, StorageAccountType.BlobOnly);
IStorageBlobClient blobClient = storageAccount.CreateBlobClient();
var container = blobClient.GetContainerReference(HostContainerNames.Hosts);
return(container);
}
public async Task <IBinding> TryCreateAsync(BindingProviderContext context)
{
ParameterInfo parameter = context.Parameter;
BlobAttribute blobAttribute = parameter.GetCustomAttribute <BlobAttribute>(inherit: false);
if (blobAttribute == null)
{
return(null);
}
string resolvedPath = Resolve(blobAttribute.BlobPath);
IBindableBlobPath path = null;
IStorageAccount account = await _accountProvider.GetStorageAccountAsync(context.Parameter, context.CancellationToken);
StorageClientFactoryContext clientFactoryContext = new StorageClientFactoryContext
{
Parameter = context.Parameter
};
IStorageBlobClient client = account.CreateBlobClient(clientFactoryContext);
// first try to bind to the Container
IArgumentBinding <IStorageBlobContainer> containerArgumentBinding = _blobContainerArgumentProvider.TryCreate(parameter);
if (containerArgumentBinding == null)
{
// if this isn't a Container binding, try a Blob binding
IBlobArgumentBinding blobArgumentBinding = _blobArgumentProvider.TryCreate(parameter, blobAttribute.Access);
if (blobArgumentBinding == null)
{
throw new InvalidOperationException("Can't bind Blob to type '" + parameter.ParameterType + "'.");
}
path = BindableBlobPath.Create(resolvedPath);
path.ValidateContractCompatibility(context.BindingDataContract);
return(new BlobBinding(parameter.Name, blobArgumentBinding, client, path));
}
path = BindableBlobPath.Create(resolvedPath, isContainerBinding: true);
path.ValidateContractCompatibility(context.BindingDataContract);
BlobContainerBinding.ValidateContainerBinding(blobAttribute, parameter.ParameterType, path);
return(new BlobContainerBinding(parameter.Name, containerArgumentBinding, client, path));
}
public void BlobPolling_IgnoresClockSkew()
{
int testScanBlobLimitPerPoll = 3;
string containerName = Path.GetRandomFileName();
IStorageAccount account = CreateFakeStorageAccount();
IStorageBlobClient client = account.CreateBlobClient();
var now = DateTimeOffset.UtcNow;
var timeMap = new Dictionary <string, DateTimeOffset>();
IStorageBlobContainer container = new SkewableFakeStorageBlobContainer(new MemoryBlobStore(), containerName, client,
(IStorageBlobResultSegment blobs) =>
{
// Simulate some extreme clock skew -- the first one's LastUpdated
// wll be 60 seconds ago and the second will be 59 seconds ago.
foreach (IStorageBlob blob in blobs.Results)
{
((FakeStorageBlobProperties)blob.Properties).LastModified = timeMap[blob.Name];
}
});
IBlobListenerStrategy product = new ScanBlobScanLogHybridPollingStrategy(new TestBlobScanInfoManager());
LambdaBlobTriggerExecutor executor = new LambdaBlobTriggerExecutor();
typeof(ScanBlobScanLogHybridPollingStrategy)
.GetField("_scanBlobLimitPerPoll", BindingFlags.Instance | BindingFlags.NonPublic)
.SetValue(product, testScanBlobLimitPerPoll);
product.Register(container, executor);
product.Start();
List <string> expectedNames = new List <string>();
expectedNames.Add(CreateAblobAndUploadToContainer(container));
timeMap[expectedNames.Single()] = now.AddSeconds(-60);
RunExecuterWithExpectedBlobs(expectedNames, product, executor);
expectedNames.Clear();
expectedNames.Add(CreateAblobAndUploadToContainer(container));
timeMap[expectedNames.Single()] = now.AddSeconds(-59);
// We should see the new item.
RunExecuterWithExpectedBlobs(expectedNames, product, executor);
}
public void TestBlobListener()
{
const string containerName = "container";
IStorageAccount account = CreateFakeStorageAccount();
IStorageBlobContainer container = account.CreateBlobClient().GetContainerReference(containerName);
IBlobListenerStrategy product = new ScanContainersStrategy();
LambdaBlobTriggerExecutor executor = new LambdaBlobTriggerExecutor();
product.Register(container, executor);
product.Start();
executor.ExecuteLambda = (_) =>
{
throw new InvalidOperationException("shouldn't be any blobs in the container");
};
product.Execute();
const string expectedBlobName = "foo1.csv";
IStorageBlockBlob blob = container.GetBlockBlobReference(expectedBlobName);
container.CreateIfNotExists();
blob.UploadText("ignore");
int count = 0;
executor.ExecuteLambda = (b) =>
{
count++;
Assert.Equal(expectedBlobName, b.Name);
return(true);
};
product.Execute();
Assert.Equal(1, count);
// Now run again; shouldn't show up.
executor.ExecuteLambda = (_) =>
{
throw new InvalidOperationException("shouldn't retrigger the same blob");
};
product.Execute();
}
internal IStorageBlobDirectory GetLockDirectory(string accountName)
{
if (string.IsNullOrEmpty(accountName))
{
accountName = ConnectionStringNames.Storage;
}
IStorageBlobDirectory storageDirectory = null;
if (!_lockDirectoryMap.TryGetValue(accountName, out storageDirectory))
{
Task <IStorageAccount> task = _accountProvider.GetAccountAsync(accountName, CancellationToken.None);
IStorageAccount storageAccount = task.Result;
IStorageBlobClient blobClient = storageAccount.CreateBlobClient();
storageDirectory = blobClient.GetContainerReference(HostContainerNames.Hosts)
.GetDirectoryReference(HostDirectoryNames.SingletonLocks);
_lockDirectoryMap[accountName] = storageDirectory;
}
return(storageDirectory);
}
public void ExecuteAsync_IfBlobDoesNotMatchPattern_ReturnsTrue()
{
// Arrange
IStorageAccount account = CreateAccount();
IStorageBlobClient client = account.CreateBlobClient();
string containerName = "container";
IStorageBlobContainer container = client.GetContainerReference(containerName);
IStorageBlobContainer otherContainer = client.GetContainerReference("other");
IBlobPathSource input = BlobPathSource.Create(containerName + "/{name}");
ITriggerExecutor <IStorageBlob> product = CreateProductUnderTest(input);
IStorageBlob blob = otherContainer.GetBlockBlobReference("nonmatch");
// Act
Task <bool> task = product.ExecuteAsync(blob, CancellationToken.None);
// Assert
Assert.True(task.Result);
}
public BlobTriggerBinding(ParameterInfo parameter,
IStorageAccount hostAccount,
IStorageAccount dataAccount,
IBlobPathSource path,
IHostIdProvider hostIdProvider,
IQueueConfiguration queueConfiguration,
JobHostBlobsConfiguration blobsConfiguration,
IWebJobsExceptionHandler exceptionHandler,
IContextSetter <IBlobWrittenWatcher> blobWrittenWatcherSetter,
IContextSetter <IMessageEnqueuedWatcher> messageEnqueuedWatcherSetter,
ISharedContextProvider sharedContextProvider,
SingletonManager singletonManager,
ILoggerFactory loggerFactory)
{
_parameter = parameter ?? throw new ArgumentNullException(nameof(parameter));
_hostAccount = hostAccount ?? throw new ArgumentNullException(nameof(hostAccount));
_dataAccount = dataAccount ?? throw new ArgumentNullException(nameof(dataAccount));
StorageClientFactoryContext context = new StorageClientFactoryContext
{
Parameter = parameter
};
_blobClient = dataAccount.CreateBlobClient(context);
_accountName = BlobClient.GetAccountName(_blobClient);
_path = path ?? throw new ArgumentNullException(nameof(path));
_hostIdProvider = hostIdProvider ?? throw new ArgumentNullException(nameof(hostIdProvider));
_queueConfiguration = queueConfiguration ?? throw new ArgumentNullException(nameof(queueConfiguration));
_blobsConfiguration = blobsConfiguration ?? throw new ArgumentNullException(nameof(blobsConfiguration));
_exceptionHandler = exceptionHandler ?? throw new ArgumentNullException(nameof(exceptionHandler));
_blobWrittenWatcherSetter = blobWrittenWatcherSetter ?? throw new ArgumentNullException(nameof(blobWrittenWatcherSetter));
_messageEnqueuedWatcherSetter = messageEnqueuedWatcherSetter ?? throw new ArgumentNullException(nameof(messageEnqueuedWatcherSetter));
_sharedContextProvider = sharedContextProvider ?? throw new ArgumentNullException(nameof(sharedContextProvider));
_singletonManager = singletonManager ?? throw new ArgumentNullException(nameof(singletonManager));
_loggerFactory = loggerFactory;
_converter = CreateConverter(_blobClient);
_bindingDataContract = CreateBindingDataContract(path);
}
public void ScanBlobScanLogHybridPollingStrategyTestBlobListener()
{
string containerName = Path.GetRandomFileName();
IStorageAccount account = CreateFakeStorageAccount();
IStorageBlobContainer container = account.CreateBlobClient().GetContainerReference(containerName);
IBlobListenerStrategy product = new ScanBlobScanLogHybridPollingStrategy(new TestBlobScanInfoManager());
LambdaBlobTriggerExecutor executor = new LambdaBlobTriggerExecutor();
product.Register(container, executor);
product.Start();
RunExecuterWithExpectedBlobs(new List <string>(), product, executor);
string expectedBlobName = CreateAblobAndUploadToContainer(container);
RunExecuterWithExpectedBlobs(new List <string>()
{
expectedBlobName
}, product, executor);
// Now run again; shouldn't show up.
RunExecuterWithExpectedBlobs(new List <string>(), product, executor);
}
internal IStorageBlobDirectory GetLockDirectory(string accountName)
{
if (string.IsNullOrEmpty(accountName))
{
accountName = ConnectionStringNames.Storage;
}
IStorageBlobDirectory storageDirectory = null;
if (!_lockDirectoryMap.TryGetValue(accountName, out storageDirectory))
{
Task <IStorageAccount> task = _accountProvider.GetStorageAccountAsync(accountName, CancellationToken.None);
IStorageAccount storageAccount = task.Result;
// singleton requires block blobs, cannot be premium
storageAccount.AssertTypeOneOf(StorageAccountType.GeneralPurpose, StorageAccountType.BlobOnly);
IStorageBlobClient blobClient = storageAccount.CreateBlobClient();
storageDirectory = blobClient.GetContainerReference(HostContainerNames.Hosts)
.GetDirectoryReference(HostDirectoryNames.SingletonLocks);
_lockDirectoryMap[accountName] = storageDirectory;
}
return(storageDirectory);
}
private static async Task ValidateCredentialsAsyncCore(IStorageAccount account,
CancellationToken cancellationToken)
{
// Verify the credentials are correct.
// Have to actually ping a storage operation.
IStorageBlobClient client = account.CreateBlobClient();
try
{
// This can hang for a long time if the account name is wrong.
// If will fail fast if the password is incorrect.
await client.GetServicePropertiesAsync(cancellationToken);
}
catch (OperationCanceledException)
{
throw;
}
catch
{
string message = String.Format(CultureInfo.CurrentCulture,
"The account credentials for '{0}' are incorrect.", account.Credentials.AccountName);
throw new InvalidOperationException(message);
}
}
private static async Task ValidateCredentialsAsyncCore(IStorageAccount account,
CancellationToken cancellationToken)
{
// Verify the credentials are correct.
// Have to actually ping a storage operation.
IStorageBlobClient client = account.CreateBlobClient();
try
{
// This can hang for a long time if the account name is wrong.
// If will fail fast if the password is incorrect.
await client.GetServicePropertiesAsync(cancellationToken);
}
catch (OperationCanceledException)
{
throw;
}
catch
{
string message = String.Format(CultureInfo.CurrentCulture,
"The account credentials for '{0}' are incorrect.", account.Credentials.AccountName);
throw new InvalidOperationException(message);
}
}
public BlobTriggerBinding(string parameterName,
IArgumentBinding <IStorageBlob> argumentBinding,
IStorageAccount account,
IBlobPathSource path,
IHostIdProvider hostIdProvider,
IQueueConfiguration queueConfiguration,
IBackgroundExceptionDispatcher backgroundExceptionDispatcher,
IContextSetter <IBlobWrittenWatcher> blobWrittenWatcherSetter,
IContextSetter <IMessageEnqueuedWatcher> messageEnqueuedWatcherSetter,
ISharedContextProvider sharedContextProvider,
TextWriter log)
{
if (argumentBinding == null)
{
throw new ArgumentNullException("argumentBinding");
}
if (account == null)
{
throw new ArgumentNullException("account");
}
if (path == null)
{
throw new ArgumentNullException("path");
}
if (hostIdProvider == null)
{
throw new ArgumentNullException("hostIdProvider");
}
if (queueConfiguration == null)
{
throw new ArgumentNullException("queueConfiguration");
}
if (backgroundExceptionDispatcher == null)
{
throw new ArgumentNullException("backgroundExceptionDispatcher");
}
if (blobWrittenWatcherSetter == null)
{
throw new ArgumentNullException("blobWrittenWatcherSetter");
}
if (messageEnqueuedWatcherSetter == null)
{
throw new ArgumentNullException("messageEnqueuedWatcherSetter");
}
if (sharedContextProvider == null)
{
throw new ArgumentNullException("sharedContextProvider");
}
if (log == null)
{
throw new ArgumentNullException("log");
}
_parameterName = parameterName;
_argumentBinding = argumentBinding;
_account = account;
_client = account.CreateBlobClient();
_accountName = BlobClient.GetAccountName(_client);
_path = path;
_hostIdProvider = hostIdProvider;
_queueConfiguration = queueConfiguration;
_backgroundExceptionDispatcher = backgroundExceptionDispatcher;
_blobWrittenWatcherSetter = blobWrittenWatcherSetter;
_messageEnqueuedWatcherSetter = messageEnqueuedWatcherSetter;
_sharedContextProvider = sharedContextProvider;
_log = log;
_converter = CreateConverter(_client);
_bindingDataContract = CreateBindingDataContract(path);
}
private static IStorageBlobContainer GetContainerReference(IStorageAccount account, string containerName)
{
IStorageBlobClient client = account.CreateBlobClient();
return(client.GetContainerReference(ContainerName));
}
public BlobTriggerBinding(ParameterInfo parameter,
IArgumentBinding<IStorageBlob> argumentBinding,
IStorageAccount hostAccount,
IStorageAccount dataAccount,
IBlobPathSource path,
IHostIdProvider hostIdProvider,
IQueueConfiguration queueConfiguration,
IBackgroundExceptionDispatcher backgroundExceptionDispatcher,
IContextSetter<IBlobWrittenWatcher> blobWrittenWatcherSetter,
IContextSetter<IMessageEnqueuedWatcher> messageEnqueuedWatcherSetter,
ISharedContextProvider sharedContextProvider,
TraceWriter trace)
{
if (parameter == null)
{
throw new ArgumentNullException("parameter");
}
if (argumentBinding == null)
{
throw new ArgumentNullException("argumentBinding");
}
if (hostAccount == null)
{
throw new ArgumentNullException("hostAccount");
}
if (dataAccount == null)
{
throw new ArgumentNullException("dataAccount");
}
if (path == null)
{
throw new ArgumentNullException("path");
}
if (hostIdProvider == null)
{
throw new ArgumentNullException("hostIdProvider");
}
if (queueConfiguration == null)
{
throw new ArgumentNullException("queueConfiguration");
}
if (backgroundExceptionDispatcher == null)
{
throw new ArgumentNullException("backgroundExceptionDispatcher");
}
if (blobWrittenWatcherSetter == null)
{
throw new ArgumentNullException("blobWrittenWatcherSetter");
}
if (messageEnqueuedWatcherSetter == null)
{
throw new ArgumentNullException("messageEnqueuedWatcherSetter");
}
if (sharedContextProvider == null)
{
throw new ArgumentNullException("sharedContextProvider");
}
if (trace == null)
{
throw new ArgumentNullException("trace");
}
_parameter = parameter;
_argumentBinding = argumentBinding;
_hostAccount = hostAccount;
_dataAccount = dataAccount;
StorageClientFactoryContext context = new StorageClientFactoryContext
{
Parameter = parameter
};
_blobClient = dataAccount.CreateBlobClient(context);
_accountName = BlobClient.GetAccountName(_blobClient);
_path = path;
_hostIdProvider = hostIdProvider;
_queueConfiguration = queueConfiguration;
_backgroundExceptionDispatcher = backgroundExceptionDispatcher;
_blobWrittenWatcherSetter = blobWrittenWatcherSetter;
_messageEnqueuedWatcherSetter = messageEnqueuedWatcherSetter;
_sharedContextProvider = sharedContextProvider;
_trace = trace;
_converter = CreateConverter(_blobClient);
_bindingDataContract = CreateBindingDataContract(path);
}
public static async Task <JobHostContext> CreateAndLogHostStartedAsync(
JobHost host,
IStorageAccountProvider storageAccountProvider,
IQueueConfiguration queueConfiguration,
ITypeLocator typeLocator,
IJobActivator activator,
INameResolver nameResolver,
IConsoleProvider consoleProvider,
JobHostConfiguration config,
CancellationToken shutdownToken,
CancellationToken cancellationToken,
IHostIdProvider hostIdProvider = null,
FunctionExecutor functionExecutor = null,
IFunctionIndexProvider functionIndexProvider = null,
IBindingProvider bindingProvider = null,
IHostInstanceLoggerProvider hostInstanceLogerProvider = null,
IFunctionInstanceLoggerProvider functionInstanceLoggerProvider = null,
IFunctionOutputLoggerProvider functionOutputLoggerProvider = null,
IBackgroundExceptionDispatcher backgroundExceptionDispatcher = null,
SingletonManager singletonManager = null)
{
if (hostIdProvider == null)
{
hostIdProvider = new DynamicHostIdProvider(storageAccountProvider, () => functionIndexProvider);
}
IExtensionTypeLocator extensionTypeLocator = new ExtensionTypeLocator(typeLocator);
if (backgroundExceptionDispatcher == null)
{
backgroundExceptionDispatcher = BackgroundExceptionDispatcher.Instance;
}
ContextAccessor <IMessageEnqueuedWatcher> messageEnqueuedWatcherAccessor = new ContextAccessor <IMessageEnqueuedWatcher>();
ContextAccessor <IBlobWrittenWatcher> blobWrittenWatcherAccessor = new ContextAccessor <IBlobWrittenWatcher>();
ISharedContextProvider sharedContextProvider = new SharedContextProvider();
// Create a wrapper TraceWriter that delegates to both the user
// TraceWriter specified on Config (if present), as well as to Console
TraceWriter trace = new ConsoleTraceWriter(config.Tracing, consoleProvider.Out);
// Register system services with the service container
config.AddService <INameResolver>(nameResolver);
ExtensionConfigContext context = new ExtensionConfigContext
{
Config = config,
Trace = trace,
Host = host
};
InvokeExtensionConfigProviders(context);
IExtensionRegistry extensions = config.GetExtensions();
ITriggerBindingProvider triggerBindingProvider = DefaultTriggerBindingProvider.Create(nameResolver,
storageAccountProvider, extensionTypeLocator, hostIdProvider, queueConfiguration, backgroundExceptionDispatcher,
messageEnqueuedWatcherAccessor, blobWrittenWatcherAccessor, sharedContextProvider, extensions, trace);
if (bindingProvider == null)
{
bindingProvider = DefaultBindingProvider.Create(nameResolver, storageAccountProvider, extensionTypeLocator, messageEnqueuedWatcherAccessor, blobWrittenWatcherAccessor, extensions);
}
DefaultLoggerProvider loggerProvider = new DefaultLoggerProvider(storageAccountProvider, trace);
if (singletonManager == null)
{
singletonManager = new SingletonManager(storageAccountProvider, backgroundExceptionDispatcher, config.Singleton, trace, config.NameResolver);
}
using (CancellationTokenSource combinedCancellationSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, shutdownToken))
{
CancellationToken combinedCancellationToken = combinedCancellationSource.Token;
await WriteSiteExtensionManifestAsync(combinedCancellationToken);
IStorageAccount dashboardAccount = await storageAccountProvider.GetDashboardAccountAsync(combinedCancellationToken);
IHostInstanceLogger hostInstanceLogger = null;
if (hostInstanceLogerProvider != null)
{
hostInstanceLogger = await hostInstanceLogerProvider.GetAsync(combinedCancellationToken);
}
else
{
hostInstanceLogger = await((IHostInstanceLoggerProvider)loggerProvider).GetAsync(combinedCancellationToken);
}
IFunctionInstanceLogger functionInstanceLogger = null;
if (functionInstanceLoggerProvider != null)
{
functionInstanceLogger = await functionInstanceLoggerProvider.GetAsync(combinedCancellationToken);
}
else
{
functionInstanceLogger = (IFunctionInstanceLogger)(await((IFunctionInstanceLoggerProvider)loggerProvider).GetAsync(combinedCancellationToken));
}
IFunctionOutputLogger functionOutputLogger = null;
if (functionOutputLoggerProvider != null)
{
functionOutputLogger = await functionOutputLoggerProvider.GetAsync(combinedCancellationToken);
}
else
{
functionOutputLogger = (IFunctionOutputLogger)(await((IFunctionOutputLoggerProvider)loggerProvider).GetAsync(combinedCancellationToken));
}
if (functionExecutor == null)
{
functionExecutor = new FunctionExecutor(functionInstanceLogger, functionOutputLogger, backgroundExceptionDispatcher, trace, config.FunctionTimeout);
}
if (functionIndexProvider == null)
{
functionIndexProvider = new FunctionIndexProvider(typeLocator, triggerBindingProvider, bindingProvider, activator, functionExecutor, extensions, singletonManager);
}
IFunctionIndex functions = await functionIndexProvider.GetAsync(combinedCancellationToken);
IListenerFactory functionsListenerFactory = new HostListenerFactory(functions.ReadAll(), singletonManager, activator, nameResolver, trace);
IFunctionExecutor hostCallExecutor;
IListener listener;
HostOutputMessage hostOutputMessage;
if (dashboardAccount != null)
{
string hostId = await hostIdProvider.GetHostIdAsync(cancellationToken);
string sharedQueueName = HostQueueNames.GetHostQueueName(hostId);
IStorageQueueClient dashboardQueueClient = dashboardAccount.CreateQueueClient();
IStorageQueue sharedQueue = dashboardQueueClient.GetQueueReference(sharedQueueName);
IListenerFactory sharedQueueListenerFactory = new HostMessageListenerFactory(sharedQueue,
queueConfiguration, backgroundExceptionDispatcher, trace, functions,
functionInstanceLogger, functionExecutor);
Guid hostInstanceId = Guid.NewGuid();
string instanceQueueName = HostQueueNames.GetHostQueueName(hostInstanceId.ToString("N"));
IStorageQueue instanceQueue = dashboardQueueClient.GetQueueReference(instanceQueueName);
IListenerFactory instanceQueueListenerFactory = new HostMessageListenerFactory(instanceQueue,
queueConfiguration, backgroundExceptionDispatcher, trace, functions,
functionInstanceLogger, functionExecutor);
HeartbeatDescriptor heartbeatDescriptor = new HeartbeatDescriptor
{
SharedContainerName = HostContainerNames.Hosts,
SharedDirectoryName = HostDirectoryNames.Heartbeats + "/" + hostId,
InstanceBlobName = hostInstanceId.ToString("N"),
ExpirationInSeconds = (int)HeartbeatIntervals.ExpirationInterval.TotalSeconds
};
IStorageBlockBlob blob = dashboardAccount.CreateBlobClient()
.GetContainerReference(heartbeatDescriptor.SharedContainerName)
.GetBlockBlobReference(heartbeatDescriptor.SharedDirectoryName + "/" + heartbeatDescriptor.InstanceBlobName);
IRecurrentCommand heartbeatCommand = new UpdateHostHeartbeatCommand(new HeartbeatCommand(blob));
IEnumerable <MethodInfo> indexedMethods = functions.ReadAllMethods();
Assembly hostAssembly = GetHostAssembly(indexedMethods);
string displayName = hostAssembly != null?hostAssembly.GetName().Name : "Unknown";
hostOutputMessage = new DataOnlyHostOutputMessage
{
HostInstanceId = hostInstanceId,
HostDisplayName = displayName,
SharedQueueName = sharedQueueName,
InstanceQueueName = instanceQueueName,
Heartbeat = heartbeatDescriptor,
WebJobRunIdentifier = WebJobRunIdentifier.Current
};
hostCallExecutor = CreateHostCallExecutor(instanceQueueListenerFactory, heartbeatCommand,
backgroundExceptionDispatcher, shutdownToken, functionExecutor);
IListenerFactory hostListenerFactory = new CompositeListenerFactory(functionsListenerFactory,
sharedQueueListenerFactory, instanceQueueListenerFactory);
listener = CreateHostListener(hostListenerFactory, heartbeatCommand, backgroundExceptionDispatcher, shutdownToken);
// Publish this to Azure logging account so that a web dashboard can see it.
await LogHostStartedAsync(functions, hostOutputMessage, hostInstanceLogger, combinedCancellationToken);
}
else
{
hostCallExecutor = new ShutdownFunctionExecutor(shutdownToken, functionExecutor);
IListener factoryListener = new ListenerFactoryListener(functionsListenerFactory);
IListener shutdownListener = new ShutdownListener(shutdownToken, factoryListener);
listener = shutdownListener;
hostOutputMessage = new DataOnlyHostOutputMessage();
}
functionExecutor.HostOutputMessage = hostOutputMessage;
IEnumerable <FunctionDescriptor> descriptors = functions.ReadAllDescriptors();
int descriptorsCount = descriptors.Count();
if (descriptorsCount == 0)
{
trace.Warning(string.Format("No job functions found. Try making your job classes and methods public. {0}",
Constants.ExtensionInitializationMessage), TraceSource.Indexing);
}
else
{
StringBuilder functionsTrace = new StringBuilder();
functionsTrace.AppendLine("Found the following functions:");
foreach (FunctionDescriptor descriptor in descriptors)
{
functionsTrace.AppendLine(descriptor.FullName);
}
trace.Info(functionsTrace.ToString(), TraceSource.Indexing);
}
return(new JobHostContext(functions, hostCallExecutor, listener, trace));
}
}
public BlobTriggerBinding(string parameterName,
IArgumentBinding<IStorageBlob> argumentBinding,
IStorageAccount account,
IBlobPathSource path,
IHostIdProvider hostIdProvider,
IQueueConfiguration queueConfiguration,
IBackgroundExceptionDispatcher backgroundExceptionDispatcher,
IContextSetter<IBlobWrittenWatcher> blobWrittenWatcherSetter,
IContextSetter<IMessageEnqueuedWatcher> messageEnqueuedWatcherSetter,
ISharedContextProvider sharedContextProvider,
TextWriter log)
{
if (argumentBinding == null)
{
throw new ArgumentNullException("argumentBinding");
}
if (account == null)
{
throw new ArgumentNullException("account");
}
if (path == null)
{
throw new ArgumentNullException("path");
}
if (hostIdProvider == null)
{
throw new ArgumentNullException("hostIdProvider");
}
if (queueConfiguration == null)
{
throw new ArgumentNullException("queueConfiguration");
}
if (backgroundExceptionDispatcher == null)
{
throw new ArgumentNullException("backgroundExceptionDispatcher");
}
if (blobWrittenWatcherSetter == null)
{
throw new ArgumentNullException("blobWrittenWatcherSetter");
}
if (messageEnqueuedWatcherSetter == null)
{
throw new ArgumentNullException("messageEnqueuedWatcherSetter");
}
if (sharedContextProvider == null)
{
throw new ArgumentNullException("sharedContextProvider");
}
if (log == null)
{
throw new ArgumentNullException("log");
}
_parameterName = parameterName;
_argumentBinding = argumentBinding;
_account = account;
_client = account.CreateBlobClient();
_accountName = BlobClient.GetAccountName(_client);
_path = path;
_hostIdProvider = hostIdProvider;
_queueConfiguration = queueConfiguration;
_backgroundExceptionDispatcher = backgroundExceptionDispatcher;
_blobWrittenWatcherSetter = blobWrittenWatcherSetter;
_messageEnqueuedWatcherSetter = messageEnqueuedWatcherSetter;
_sharedContextProvider = sharedContextProvider;
_log = log;
_converter = CreateConverter(_client);
_bindingDataContract = CreateBindingDataContract(path);
}
/// <summary>Initializes a new instance of the <see cref="HeartbeatCommand"/> class.</summary>
/// <param name="account">The storage account in which to write the heartbeat.</param>
/// <param name="containerName">The name of the container in which to write the heartbeat.</param>
/// <param name="blobName">The name of the heartbeat blob (including the directory name, if any).</param>
public HeartbeatCommand(IStorageAccount account, string containerName, string blobName)
: this(account.CreateBlobClient().GetContainerReference(containerName).GetBlockBlobReference(blobName))
{
}
// Test that the credentials are valid and classify the account.Type as one of StorageAccountTypes
private static async Task ValidateCredentialsAsyncCore(IStorageAccount account, CancellationToken cancellationToken)
{
// Verify the credentials are correct.
// Have to actually ping a storage operation.
IStorageBlobClient client = account.CreateBlobClient();
try
{
// This can hang for a long time if the account name is wrong.
// If will fail fast if the password is incorrect.
await client.GetServicePropertiesAsync(cancellationToken);
}
catch (OperationCanceledException)
{
throw;
}
catch (Exception e)
{
var storageException = e as StorageException;
if (storageException?.RequestInformation?.HttpStatusCode == 400 &&
storageException?.RequestInformation?.ExtendedErrorInformation?.ErrorCode == "InvalidQueryParameterValue")
{
// Premium storage accounts do not support the GetServicePropertiesAsync call, and respond with a 400 'InvalidQueryParameterValue'.
// If we see this error response classify the account as a premium account
account.Type = StorageAccountType.Premium;
return;
}
else
{
// If not a recognized error, the credentials are invalid
string message = String.Format(CultureInfo.CurrentCulture,
"Invalid storage account '{0}'. Please make sure your credentials are correct.",
account.Credentials.AccountName);
throw new InvalidOperationException(message);
}
}
IStorageQueueClient queueClient = account.CreateQueueClient();
IStorageQueue queue = queueClient.GetQueueReference("name");
try
{
await queue.ExistsAsync(cancellationToken);
}
catch (OperationCanceledException)
{
throw;
}
catch (StorageException exception)
{
WebException webException = exception.GetBaseException() as WebException;
if (webException?.Status == WebExceptionStatus.NameResolutionFailure)
{
// Blob-only storage accounts do not support services other than Blob.
// If we see a name resolution failure on the queue endpoint classify as a blob-only account
account.Type = StorageAccountType.BlobOnly;
return;
}
throw;
}
}
// Do the full runtime intitialization. This includes static initialization.
// This mainly means:
// - indexing the functions
// - spinning up the listeners (so connecting to the services)
public static async Task <JobHostContext> CreateJobHostContextAsync(
this JobHostConfiguration config,
ServiceProviderWrapper services, // Results from first phase
JobHost host,
CancellationToken shutdownToken,
CancellationToken cancellationToken)
{
FunctionExecutor functionExecutor = services.GetService <FunctionExecutor>();
IFunctionIndexProvider functionIndexProvider = services.GetService <IFunctionIndexProvider>();
ITriggerBindingProvider triggerBindingProvider = services.GetService <ITriggerBindingProvider>();
IBindingProvider bindingProvider = services.GetService <IBindingProvider>();
SingletonManager singletonManager = services.GetService <SingletonManager>();
IJobActivator activator = services.GetService <IJobActivator>();
IHostIdProvider hostIdProvider = services.GetService <IHostIdProvider>();
INameResolver nameResolver = services.GetService <INameResolver>();
IExtensionRegistry extensions = services.GetExtensions();
IStorageAccountProvider storageAccountProvider = services.GetService <IStorageAccountProvider>();
ILoggerFactory loggerFactory = services.GetService <ILoggerFactory>();
IFunctionResultAggregatorFactory aggregatorFactory = services.GetService <IFunctionResultAggregatorFactory>();
IAsyncCollector <FunctionInstanceLogEntry> functionEventCollector = null;
// Create the aggregator if all the pieces are configured
IAsyncCollector <FunctionInstanceLogEntry> aggregator = null;
if (loggerFactory != null && aggregatorFactory != null && config.Aggregator.IsEnabled)
{
aggregator = aggregatorFactory.Create(config.Aggregator.BatchSize, config.Aggregator.FlushTimeout, loggerFactory);
}
IQueueConfiguration queueConfiguration = services.GetService <IQueueConfiguration>();
var blobsConfiguration = config.Blobs;
TraceWriter trace = services.GetService <TraceWriter>();
IAsyncCollector <FunctionInstanceLogEntry> registeredFunctionEventCollector = services.GetService <IAsyncCollector <FunctionInstanceLogEntry> >();
if (registeredFunctionEventCollector != null && aggregator != null)
{
// If there are both an aggregator and a registered FunctionEventCollector, wrap them in a composite
functionEventCollector = new CompositeFunctionEventCollector(new[] { registeredFunctionEventCollector, aggregator });
}
else
{
// Otherwise, take whichever one is null (or use null if both are)
functionEventCollector = aggregator ?? registeredFunctionEventCollector;
}
IWebJobsExceptionHandler exceptionHandler = services.GetService <IWebJobsExceptionHandler>();
if (exceptionHandler != null)
{
exceptionHandler.Initialize(host);
}
bool hasFastTableHook = services.GetService <IAsyncCollector <FunctionInstanceLogEntry> >() != null;
bool noDashboardStorage = config.DashboardConnectionString == null;
// Only testing will override these interfaces.
IHostInstanceLoggerProvider hostInstanceLoggerProvider = services.GetService <IHostInstanceLoggerProvider>();
IFunctionInstanceLoggerProvider functionInstanceLoggerProvider = services.GetService <IFunctionInstanceLoggerProvider>();
IFunctionOutputLoggerProvider functionOutputLoggerProvider = services.GetService <IFunctionOutputLoggerProvider>();
if (hostInstanceLoggerProvider == null && functionInstanceLoggerProvider == null && functionOutputLoggerProvider == null)
{
if (hasFastTableHook && noDashboardStorage)
{
var loggerProvider = new FastTableLoggerProvider(trace);
hostInstanceLoggerProvider = loggerProvider;
functionInstanceLoggerProvider = loggerProvider;
functionOutputLoggerProvider = loggerProvider;
}
else
{
var loggerProvider = new DefaultLoggerProvider(storageAccountProvider, trace);
hostInstanceLoggerProvider = loggerProvider;
functionInstanceLoggerProvider = loggerProvider;
functionOutputLoggerProvider = loggerProvider;
}
}
using (CancellationTokenSource combinedCancellationSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, shutdownToken))
{
CancellationToken combinedCancellationToken = combinedCancellationSource.Token;
await WriteSiteExtensionManifestAsync(combinedCancellationToken);
IStorageAccount dashboardAccount = await storageAccountProvider.GetDashboardAccountAsync(combinedCancellationToken);
IHostInstanceLogger hostInstanceLogger = await hostInstanceLoggerProvider.GetAsync(combinedCancellationToken);
IFunctionInstanceLogger functionInstanceLogger = await functionInstanceLoggerProvider.GetAsync(combinedCancellationToken);
IFunctionOutputLogger functionOutputLogger = await functionOutputLoggerProvider.GetAsync(combinedCancellationToken);
if (functionExecutor == null)
{
var extensionRegistry = config.GetService <IExtensionRegistry>();
var globalFunctionFilters = extensionRegistry.GetFunctionFilters();
functionExecutor = new FunctionExecutor(functionInstanceLogger, functionOutputLogger, exceptionHandler, trace, functionEventCollector, loggerFactory, globalFunctionFilters);
services.AddService(functionExecutor);
}
if (functionIndexProvider == null)
{
functionIndexProvider = new FunctionIndexProvider(
services.GetService <ITypeLocator>(),
triggerBindingProvider,
bindingProvider,
activator,
functionExecutor,
extensions,
singletonManager,
trace,
loggerFactory);
// Important to set this so that the func we passed to DynamicHostIdProvider can pick it up.
services.AddService <IFunctionIndexProvider>(functionIndexProvider);
}
IFunctionIndex functions = await functionIndexProvider.GetAsync(combinedCancellationToken);
IListenerFactory functionsListenerFactory = new HostListenerFactory(functions.ReadAll(), singletonManager, activator, nameResolver, trace, loggerFactory);
IFunctionExecutor hostCallExecutor;
IListener listener;
HostOutputMessage hostOutputMessage;
string hostId = await hostIdProvider.GetHostIdAsync(cancellationToken);
if (string.Compare(config.HostId, hostId, StringComparison.OrdinalIgnoreCase) != 0)
{
// if this isn't a static host ID, provide the HostId on the config
// so it is accessible
config.HostId = hostId;
}
if (dashboardAccount == null)
{
hostCallExecutor = new ShutdownFunctionExecutor(shutdownToken, functionExecutor);
IListener factoryListener = new ListenerFactoryListener(functionsListenerFactory);
IListener shutdownListener = new ShutdownListener(shutdownToken, factoryListener);
listener = shutdownListener;
hostOutputMessage = new DataOnlyHostOutputMessage();
}
else
{
string sharedQueueName = HostQueueNames.GetHostQueueName(hostId);
IStorageQueueClient dashboardQueueClient = dashboardAccount.CreateQueueClient();
IStorageQueue sharedQueue = dashboardQueueClient.GetQueueReference(sharedQueueName);
IListenerFactory sharedQueueListenerFactory = new HostMessageListenerFactory(sharedQueue,
queueConfiguration, exceptionHandler, trace, loggerFactory, functions,
functionInstanceLogger, functionExecutor);
Guid hostInstanceId = Guid.NewGuid();
string instanceQueueName = HostQueueNames.GetHostQueueName(hostInstanceId.ToString("N"));
IStorageQueue instanceQueue = dashboardQueueClient.GetQueueReference(instanceQueueName);
IListenerFactory instanceQueueListenerFactory = new HostMessageListenerFactory(instanceQueue,
queueConfiguration, exceptionHandler, trace, loggerFactory, functions,
functionInstanceLogger, functionExecutor);
HeartbeatDescriptor heartbeatDescriptor = new HeartbeatDescriptor
{
SharedContainerName = HostContainerNames.Hosts,
SharedDirectoryName = HostDirectoryNames.Heartbeats + "/" + hostId,
InstanceBlobName = hostInstanceId.ToString("N"),
ExpirationInSeconds = (int)HeartbeatIntervals.ExpirationInterval.TotalSeconds
};
IStorageBlockBlob blob = dashboardAccount.CreateBlobClient()
.GetContainerReference(heartbeatDescriptor.SharedContainerName)
.GetBlockBlobReference(heartbeatDescriptor.SharedDirectoryName + "/" + heartbeatDescriptor.InstanceBlobName);
IRecurrentCommand heartbeatCommand = new UpdateHostHeartbeatCommand(new HeartbeatCommand(blob));
IEnumerable <MethodInfo> indexedMethods = functions.ReadAllMethods();
Assembly hostAssembly = GetHostAssembly(indexedMethods);
string displayName = hostAssembly != null?AssemblyNameCache.GetName(hostAssembly).Name : "Unknown";
hostOutputMessage = new DataOnlyHostOutputMessage
{
HostInstanceId = hostInstanceId,
HostDisplayName = displayName,
SharedQueueName = sharedQueueName,
InstanceQueueName = instanceQueueName,
Heartbeat = heartbeatDescriptor,
WebJobRunIdentifier = WebJobRunIdentifier.Current
};
hostCallExecutor = CreateHostCallExecutor(instanceQueueListenerFactory, heartbeatCommand,
exceptionHandler, shutdownToken, functionExecutor);
IListenerFactory hostListenerFactory = new CompositeListenerFactory(functionsListenerFactory,
sharedQueueListenerFactory, instanceQueueListenerFactory);
listener = CreateHostListener(hostListenerFactory, heartbeatCommand, exceptionHandler, shutdownToken);
// Publish this to Azure logging account so that a web dashboard can see it.
await LogHostStartedAsync(functions, hostOutputMessage, hostInstanceLogger, combinedCancellationToken);
}
functionExecutor.HostOutputMessage = hostOutputMessage;
IEnumerable <FunctionDescriptor> descriptors = functions.ReadAllDescriptors();
int descriptorsCount = descriptors.Count();
ILogger startupLogger = loggerFactory?.CreateLogger(LogCategories.Startup);
if (config.UsingDevelopmentSettings)
{
string msg = "Development settings applied";
trace.Verbose(msg);
startupLogger?.LogDebug(msg);
}
if (descriptorsCount == 0)
{
string msg = string.Format("No job functions found. Try making your job classes and methods public. {0}",
Constants.ExtensionInitializationMessage);
trace.Warning(msg, Host.TraceSource.Indexing);
startupLogger?.LogWarning(msg);
}
else
{
StringBuilder functionsTrace = new StringBuilder();
functionsTrace.AppendLine("Found the following functions:");
foreach (FunctionDescriptor descriptor in descriptors)
{
functionsTrace.AppendLine(descriptor.FullName);
}
string msg = functionsTrace.ToString();
trace.Info(msg, Host.TraceSource.Indexing);
startupLogger?.LogInformation(msg);
}
return(new JobHostContext(
functions,
hostCallExecutor,
listener,
trace,
functionEventCollector,
loggerFactory));
}
}
public BlobTriggerBinding(ParameterInfo parameter,
IArgumentBinding <IStorageBlob> argumentBinding,
IStorageAccount hostAccount,
IStorageAccount dataAccount,
IBlobPathSource path,
IHostIdProvider hostIdProvider,
IQueueConfiguration queueConfiguration,
IBackgroundExceptionDispatcher backgroundExceptionDispatcher,
IContextSetter <IBlobWrittenWatcher> blobWrittenWatcherSetter,
IContextSetter <IMessageEnqueuedWatcher> messageEnqueuedWatcherSetter,
ISharedContextProvider sharedContextProvider,
SingletonManager singletonManager,
TraceWriter trace)
{
if (parameter == null)
{
throw new ArgumentNullException("parameter");
}
if (argumentBinding == null)
{
throw new ArgumentNullException("argumentBinding");
}
if (hostAccount == null)
{
throw new ArgumentNullException("hostAccount");
}
if (dataAccount == null)
{
throw new ArgumentNullException("dataAccount");
}
if (path == null)
{
throw new ArgumentNullException("path");
}
if (hostIdProvider == null)
{
throw new ArgumentNullException("hostIdProvider");
}
if (queueConfiguration == null)
{
throw new ArgumentNullException("queueConfiguration");
}
if (backgroundExceptionDispatcher == null)
{
throw new ArgumentNullException("backgroundExceptionDispatcher");
}
if (blobWrittenWatcherSetter == null)
{
throw new ArgumentNullException("blobWrittenWatcherSetter");
}
if (messageEnqueuedWatcherSetter == null)
{
throw new ArgumentNullException("messageEnqueuedWatcherSetter");
}
if (sharedContextProvider == null)
{
throw new ArgumentNullException("sharedContextProvider");
}
if (singletonManager == null)
{
throw new ArgumentNullException("singletonManager");
}
if (trace == null)
{
throw new ArgumentNullException("trace");
}
_parameter = parameter;
_argumentBinding = argumentBinding;
_hostAccount = hostAccount;
_dataAccount = dataAccount;
StorageClientFactoryContext context = new StorageClientFactoryContext
{
Parameter = parameter
};
_blobClient = dataAccount.CreateBlobClient(context);
_accountName = BlobClient.GetAccountName(_blobClient);
_path = path;
_hostIdProvider = hostIdProvider;
_queueConfiguration = queueConfiguration;
_backgroundExceptionDispatcher = backgroundExceptionDispatcher;
_blobWrittenWatcherSetter = blobWrittenWatcherSetter;
_messageEnqueuedWatcherSetter = messageEnqueuedWatcherSetter;
_sharedContextProvider = sharedContextProvider;
_singletonManager = singletonManager;
_trace = trace;
_converter = CreateConverter(_blobClient);
_bindingDataContract = CreateBindingDataContract(path);
}
/// <summary>Initializes a new instance of the <see cref="HeartbeatCommand"/> class.</summary>
/// <param name="account">The storage account in which to write the heartbeat.</param>
/// <param name="containerName">The name of the container in which to write the heartbeat.</param>
/// <param name="blobName">The name of the heartbeat blob (including the directory name, if any).</param>
public HeartbeatCommand(IStorageAccount account, string containerName, string blobName)
: this(account.CreateBlobClient().GetContainerReference(containerName).GetBlockBlobReference(blobName))
{
}