/// <summary>
/// Implements a callback delegate which will be invoked whenever there is no more work available on the queue.
/// </summary>
/// <param name="sender">The source of the event.</param>
/// <param name="idleCount">The value indicating how many times the queue listener has been idle.</param>
/// <param name="delay">The time interval during which the queue listener will be instructed to sleep before performing next unit of work.</param>
/// <returns>A boolean flag indicating that the queue listener should stop processing any further work and must terminate.</returns>
private bool HandleQueueEmptyEvent(object sender, int idleCount, out TimeSpan delay)
{
// The sender is an instance of the ICloudQueueServiceWorkerRoleExtension, we can safely perform type casting.
ICloudQueueServiceWorkerRoleExtension queueService = sender as ICloudQueueServiceWorkerRoleExtension;
// Find out which extension is responsible for retrieving the worker role configuration settings.
IWorkItemProcessorConfigurationExtension config = Extensions.Find <IWorkItemProcessorConfigurationExtension>();
// Get the current state of the queue listener to determine point-in-time load characteristics.
CloudQueueListenerInfo queueServiceState = queueService.QueryState();
// Set up the initial parameters, read configuration settings.
int deltaBackoffMs = 100;
int minimumIdleIntervalMs = Convert.ToInt32(config.Settings.MinimumIdleInterval.TotalMilliseconds);
int maximumIdleIntervalMs = Convert.ToInt32(config.Settings.MaximumIdleInterval.TotalMilliseconds);
// Calculate a new sleep interval value that will follow a random exponential back-off curve.
int delta = (int)((Math.Pow(2.0, (double)idleCount) - 1.0) * (new Random()).Next((int)(deltaBackoffMs * 0.8), (int)(deltaBackoffMs * 1.2)));
int interval = Math.Min(minimumIdleIntervalMs + delta, maximumIdleIntervalMs);
// Pass the calculated interval to the dequeue task to enable it to enter into a sleep state for the specified duration.
delay = TimeSpan.FromMilliseconds((double)interval);
// As soon as interval reaches its maximum, tell the source dequeue task that it must gracefully terminate itself
// unless this is a last deqeueue task. If so, we are not going to keep it running and continue polling the queue.
return(delay.TotalMilliseconds >= maximumIdleIntervalMs);
}
/// <summary>
/// Notifies this extension component that it has been registered in the owner's collection of extensions.
/// </summary>
/// <param name="owner">The extensible owner object that aggregates this extension.</param>
public void Attach(IExtensibleCloudServiceComponent owner)
{
owner.Extensions.Demand <IWorkItemProcessorConfigurationExtension>();
owner.Extensions.Demand <IRoleConfigurationSettingsExtension>();
owner.Extensions.Demand <ICloudCacheProviderExtension>();
owner.Extensions.Demand <IRulesEngineServiceClientExtension>();
this.configSettingsExtension = owner.Extensions.Find <IWorkItemProcessorConfigurationExtension>();
this.roleConfigExtension = owner.Extensions.Find <IRoleConfigurationSettingsExtension>();
this.cacheProviderExtension = owner.Extensions.Find <ICloudCacheProviderExtension>();
this.rulesEngineExtension = owner.Extensions.Find <IRulesEngineServiceClientExtension>();
}
private void HandlePersistenceQueueItem(PersistenceQueueItemInfo itemInfo)
{
Guard.ArgumentNotNull(itemInfo, "itemInfo");
var callToken = TraceManager.WorkerRoleComponent.TraceIn(itemInfo.QueueItemId, itemInfo.QueueItemType, itemInfo.QueueItemSize);
try
{
this.owner.Extensions.Demand <IRulesEngineServiceClientExtension>();
this.owner.Extensions.Demand <IWorkItemProcessorConfigurationExtension>();
IWorkItemProcessorConfigurationExtension configSettingsExtension = this.owner.Extensions.Find <IWorkItemProcessorConfigurationExtension>();
IRulesEngineServiceClientExtension rulesEngineExtension = this.owner.Extensions.Find <IRulesEngineServiceClientExtension>();
// Invoke a policy to determine the number of dequeue listeners required to process this volume of data.
StringDictionaryFact policyExecutionResult = rulesEngineExtension.ExecutePolicy <StringDictionaryFact>(configSettingsExtension.Settings.HandlingPolicyName, itemInfo);
int dequeueTaskCount = policyExecutionResult != null && !String.IsNullOrEmpty(policyExecutionResult.Items[Resources.StringDictionaryFactValueDequeueListenerCount]) ? Convert.ToInt32(policyExecutionResult.Items[Resources.StringDictionaryFactValueDequeueListenerCount]) : 0;
// Check if the policy has provided us with the number of queue listeners.
if (dequeueTaskCount > 0)
{
// Write the number of dequeue tasks into trace log.
TraceManager.WorkerRoleComponent.TraceInfo(TraceLogMessages.DequeueTaskCountInfo, itemInfo.QueueItemType, itemInfo.QueueItemSize, dequeueTaskCount);
// Request all queue listeners to increase the number of listeners accordingly to the given workload.
foreach (var queueService in this.owner.Extensions.FindAll <ICloudQueueServiceWorkerRoleExtension>())
{
// Get the queue listener state information such as queue depth, number of activity listener threads and total number of listener threads.
var state = queueService.QueryState();
// If the queue listener count is less than suggested by the policy, start as many extra listener threads as needed.
if (state.ActiveDequeueTasks < dequeueTaskCount)
{
queueService.StartListener(dequeueTaskCount - state.ActiveDequeueTasks);
}
}
}
}
finally
{
TraceManager.WorkerRoleComponent.TraceOut(callToken);
}
}
/// <summary>
/// Extends the OnStart phase that is called by Windows Azure runtime to initialize the role instance.
/// </summary>
/// <returns>True if initialization succeeds, otherwise false.</returns>
protected override bool OnRoleStart()
{
this.EnsureExists <WorkItemProcessorConfigurationExtension>();
this.EnsureExists <WorkQueueLocationResolverExtension>();
this.EnsureExists <ScalableTransformServiceClientExtension>();
this.EnsureExists <EndpointConfigurationDiscoveryExtension>();
this.EnsureExists <RulesEngineServiceClientExtension>();
this.EnsureExists <InterRoleEventSubscriberExtension>();
IWorkItemProcessorConfigurationExtension configSettingsExtension = Extensions.Find <IWorkItemProcessorConfigurationExtension>();
IEndpointConfigurationDiscoveryExtension discoveryExtension = Extensions.Find <IEndpointConfigurationDiscoveryExtension>();
IInterRoleCommunicationExtension interCommExtension = Extensions.Find <IInterRoleCommunicationExtension>();
IInterRoleEventSubscriberExtension interCommSubscriber = Extensions.Find <IInterRoleEventSubscriberExtension>();
CloudQueueLocation inputQueueLocation = new CloudQueueLocation()
{
StorageAccount = configSettingsExtension.Settings.CloudStorageAccount,
QueueName = configSettingsExtension.Settings.WorkItemQueue,
VisibilityTimeout = configSettingsExtension.Settings.WorkItemQueueVisibilityTimeout
};
CloudQueueLocation outputQueueLocation = new CloudQueueLocation()
{
StorageAccount = configSettingsExtension.Settings.CloudStorageAccount,
QueueName = configSettingsExtension.Settings.OutputQueue,
VisibilityTimeout = configSettingsExtension.Settings.OutputQueueVisibilityTimeout
};
// Instantiate queue listeners.
var inputQueueListener = new CloudQueueListenerExtension <XDocument>(inputQueueLocation);
var outputQueueListener = new CloudQueueListenerExtension <XDocument>(outputQueueLocation);
// Configure the input queue listener.
inputQueueListener.QueueEmpty += HandleQueueEmptyEvent;
inputQueueListener.DequeueBatchSize = configSettingsExtension.Settings.DequeueBatchSize;
inputQueueListener.DequeueInterval = configSettingsExtension.Settings.MinimumIdleInterval;
// Configure the output queue listener.
outputQueueListener.QueueEmpty += HandleQueueEmptyEvent;
outputQueueListener.DequeueBatchSize = configSettingsExtension.Settings.DequeueBatchSize;
outputQueueListener.DequeueInterval = configSettingsExtension.Settings.MinimumIdleInterval;
// Instantiate queue subscribers.
InputQueueSubscriberExtension inputQueueSubscriber = new InputQueueSubscriberExtension();
OutputQueueSubscriberExtension outputQueueSubscriber = new OutputQueueSubscriberExtension();
// Register subscribers with queue listeners.
inputQueueListener.Subscribe(inputQueueSubscriber);
outputQueueListener.Subscribe(outputQueueSubscriber);
// Register custom extensions for this worker role.
Extensions.Add(inputQueueSubscriber);
Extensions.Add(outputQueueSubscriber);
Extensions.Add(outputQueueListener);
Extensions.Add(inputQueueListener);
var contractTypeMatchCondition = ServiceEndpointDiscoveryCondition.ContractTypeExactMatch(typeof(IScalableTransformationServiceContract));
var bindingTypeMatchCondition = ServiceEndpointDiscoveryCondition.BindingTypeExactMatch(typeof(NetTcpRelayBinding));
discoveryExtension.RegisterDiscoveryAction(new[] { contractTypeMatchCondition, bindingTypeMatchCondition }, (endpoint) =>
{
NetTcpRelayBinding relayBinding = endpoint.Binding as NetTcpRelayBinding;
if (relayBinding != null)
{
relayBinding.TransferMode = TransferMode.Streamed;
}
});
// Register a subscriber for all inter-role communication events.
if (interCommExtension != null && interCommSubscriber != null)
{
this.interCommSubscription = interCommExtension.Subscribe(interCommSubscriber);
}
return(true);
}
/// <summary>
/// Notifies this extension component that it has been registered in the owner's collection of extensions.
/// </summary>
/// <param name="owner">The extensible owner object that aggregates this extension.</param>
public void Attach(IExtensibleCloudServiceComponent owner)
{
this.configSettingsExtension = owner.Extensions.Find <IWorkItemProcessorConfigurationExtension>();
}
