public async Task <Task> DispatchAsync(IList <OutputMessage> outputMessages)
{
var operation = nameof(this.DispatchAsync);
var queueLength = this.pushWorkers.Sum((t) => t.QueueLength);
if (queueLength > this.setting.PartitionSetting.MaxPushQueueLength)
{
MessageDispatcherEventSource.Current.Info(MessageDispatcherEventSource.EmptyTrackingId, this, operation + ".Wait", OperationStates.Starting, $"while queueLength: {queueLength} le maxQueueLength: {this.setting.PartitionSetting.MaxPushQueueLength}");
do
{
await Task.Delay(TimeSpan.FromMilliseconds(100), this.CancelToken);
queueLength = this.pushWorkers.Sum((t) => t.QueueLength);
}while (queueLength > this.setting.PartitionSetting.MaxPushQueueLength);
MessageDispatcherEventSource.Current.Info(MessageDispatcherEventSource.EmptyTrackingId, this, operation + ".Wait", OperationStates.Succeeded, $"queueLength: {queueLength} le maxQueueLength: {this.setting.PartitionSetting.MaxPushQueueLength}");
}
var pushWorkerIndex = this.random.Next(0, this.setting.PushWorkerCount);
var tasks = new Task[outputMessages.Count];
var batches = this.Split(outputMessages, this.setting.PushWorkerCount, tasks);
for (var index = 0; index < batches.Length && !this.CancelToken.IsCancellationRequested; index++)
{
var batch = batches[index];
if (batch.Count > 0)
{
var pushWorker = this.pushWorkers[pushWorkerIndex];
if (!pushWorker.TryAdd(batch))
{
// False if cancelled
break;
}
pushWorkerIndex++;
}
if (pushWorkerIndex == this.setting.PushWorkerCount)
{
pushWorkerIndex = 0;
}
}
if (this.CancelToken.IsCancellationRequested)
{
for (var batchIndex = 0; batchIndex < batches.Length; batchIndex++)
{
List <PushTaskInfo> batch = batches[batchIndex];
for (var taskIndex = 0; taskIndex < batch.Count; taskIndex++)
{
PushTaskInfo taskInfo = batch[taskIndex];
taskInfo.Tcs.TrySetCanceled();
}
}
}
return(Task.WhenAll(tasks));
}
private List <PushTaskInfo>[] Split(IList <OutputMessage> outputMessages, int batchCount, Task[] tasks)
{
var batches = new List <PushTaskInfo> [Math.Min(batchCount, outputMessages.Count)];
for (var i = 0; i < batches.Length; i++)
{
batches[i] = new List <PushTaskInfo>();
}
var batchIndex = 0;
var taskIndex = 0;
var now = DateTime.UtcNow;
var postponed = new List <PushTaskInfo>();
for (var i = 0; i < outputMessages.Count; i++)
{
var batch = batches[batchIndex];
var outputMessage = outputMessages[i];
var pushTaskInfo = new PushTaskInfo()
{
OutputMessage = outputMessage,
Tcs = new TaskCompletionSource <bool>(TaskCreationOptions.RunContinuationsAsynchronously),
PushEnqueueTimestamp = DateTime.UtcNow
};
var expiration = outputMessage.MessageInfo.SendTime + this.setting.PartitionSetting.ServiceConfigureSetting.EventTimeToLive;
bool expired = expiration < now;
if (expired || outputMessage.Delivered || outputMessage.State == OutputMessageState.Unfilterable)
{
MessageDispatcherEventSource.Current.Info(MessageDispatcherEventSource.EmptyTrackingId, this, nameof(this.Split), OperationStates.Dropped, $"msgId={outputMessage.MessageInfo.MessageId}, batchId={outputMessage.Id}, state={outputMessage.State}, deliveryCount={outputMessage.DeliveryCount}, delivered={outputMessage.Delivered}, requestExpiration={outputMessage.RequestExpiration}, publishTime={outputMessage.MessageInfo.SendTime}, expired={expired}, expiration={expiration}, timeLived={now - outputMessage.MessageInfo.SendTime}");
pushTaskInfo.Tcs.TrySetResult(true);
}
else if (outputMessage.State == OutputMessageState.Filtered)
{
batch.Add(pushTaskInfo);
batchIndex++;
}
else
{
pushTaskInfo.OutputMessage.DeliveryTime = DateTime.UtcNow.Add(this.outputMessageQueue.Setting.RetryDelay);
postponed.Add(pushTaskInfo);
}
tasks[taskIndex] = pushTaskInfo.Tcs.Task;
taskIndex++;
if (batchIndex == batchCount)
{
batchIndex = 0;
}
}
if (postponed.Count > 0)
{
TaskHelper.FireAndForget(() => this.PostponedAsync(postponed), ex => this.UnhandledException(ex, nameof(this.PostponedAsync)));
}
return(batches);
}
private async Task <DeliveryResponse> DeliverAsync(PushTaskInfo taskInfo, DateTime now)
{
DeliveryRequest deliveryRequest = null;
DeliveryResponse deliveryResponse = new DeliveryResponse(RequestOutcome.UNKNOWN);
Exception exception = null;
try
{
deliveryRequest = new DeliveryRequest(taskInfo.OutputMessage, now + PostExpirationTimeSpan);
MessageDispatcherEventSource.Current.Info(taskInfo.OutputMessage.MessageInfo.TrackingId, this, nameof(this.DeliverAsync), OperationStates.Starting, $"msgId={taskInfo.OutputMessage.MessageInfo.MessageId}, batchId={taskInfo.OutputMessage.Id}, delivered={taskInfo.OutputMessage.Delivered}, requestExpiration={taskInfo.OutputMessage.RequestExpiration}");
var pushConnector = PushConnector.Create(taskInfo.OutputMessage.ConnectorCredential);
deliveryResponse = await pushConnector.DeliverAsync(deliveryRequest, this.CancelToken).ConfigureAwait(false);
}
catch (SocketException ex)
{
exception = ex;
deliveryResponse.DeliveryOutcome = RequestOutcome.FAILED_OPERATOR;
deliveryResponse.DeliveryDetail = ex.Message;
}
catch (OperationCanceledException ex)
{
exception = ex;
deliveryResponse.DeliveryOutcome = RequestOutcome.CANCELLED;
deliveryResponse.DeliveryDetail = ex.Message;
}
catch (Exception ex)
{
exception = ex;
deliveryResponse.DeliveryOutcome = RequestOutcome.FAILED_UNKNOWN;
deliveryResponse.DeliveryDetail = ex.Message;
}
if (deliveryResponse.DeliveryOutcome == RequestOutcome.SUCCESS)
{
deliveryRequest.Succeed();
MessageDispatcherEventSource.Current.Info(taskInfo.OutputMessage.MessageInfo.TrackingId, this, nameof(this.DeliverAsync), OperationStates.Succeeded, $"msgId={taskInfo.OutputMessage.MessageInfo.MessageId}, batchId={taskInfo.OutputMessage.Id}, E2ELantency={DateTime.UtcNow - taskInfo.OutputMessage.MessageInfo.SendTime}, PushLatency={DateTime.UtcNow - deliveryRequest.DeliverRequestTimestamp}, delivered={taskInfo.OutputMessage.Delivered}, requestExpiration={taskInfo.OutputMessage.RequestExpiration}, deliveryResponse={deliveryResponse}");
}
else
{
if (deliveryRequest != null)
{
deliveryRequest.Failed();
if (deliveryResponse.DeliveryOutcome == RequestOutcome.FAILED_UNAUTHORIZED)
{
// Need to refilter if authentication failed
taskInfo.OutputMessage.State = OutputMessageState.Nonfiltered;
}
else
{
taskInfo.OutputMessage.State = OutputMessageState.FilteredFailingDelivery;
}
MessageDispatcherEventSource.Current.Error(taskInfo.OutputMessage.MessageInfo.TrackingId, this, nameof(this.DeliverAsync), OperationStates.Failed, $"msgId={taskInfo.OutputMessage.MessageInfo.MessageId}, batchId={taskInfo.OutputMessage.Id}, outcome={deliveryResponse.DeliveryOutcome}, latency={DateTime.UtcNow - deliveryRequest.DeliverRequestTimestamp}, delivered={taskInfo.OutputMessage.Delivered}, requestExpiration={taskInfo.OutputMessage.RequestExpiration}, deliveryResponse={deliveryResponse}, exception={exception}");
}
else
{
MessageDispatcherEventSource.Current.Error(taskInfo.OutputMessage.MessageInfo.TrackingId, this, nameof(this.DeliverAsync), OperationStates.Failed, $"msgId={taskInfo.OutputMessage.MessageInfo.MessageId}, batchId={taskInfo.OutputMessage.Id}, outcome={deliveryResponse.DeliveryOutcome}, timestamp={DateTime.UtcNow}, delivered={taskInfo.OutputMessage.Delivered}, requestExpiration={taskInfo.OutputMessage.RequestExpiration}, deliveryResponse={deliveryResponse}, exception={exception}");
}
}
return(deliveryResponse);
}
private async Task StartDeliveryTask(PushTaskInfo taskInfo)
{
var operation = nameof(this.StartDeliveryTask);
try
{
var response = new DeliveryResponse(RequestOutcome.UNKNOWN);
var outputMessage = taskInfo.OutputMessage;
var now = DateTime.UtcNow;
var expiration = outputMessage.MessageInfo.SendTime + this.setting.PartitionSetting.ServiceConfigureSetting.EventTimeToLive;
var expired = expiration < now;
if (expired)
{
MessageDispatcherEventSource.Current.Info(taskInfo.OutputMessage.MessageInfo.TrackingId, this, operation + ".Deliver", OperationStates.Dropped, $"msgId={outputMessage.MessageInfo.MessageId}, batchId={outputMessage.Id}, state={outputMessage.State}, deliveryCount={outputMessage.DeliveryCount}, delivered={outputMessage.Delivered}, requestExpiration={outputMessage.RequestExpiration}, SendTime={outputMessage.MessageInfo.SendTime}, expired={expired}, expiration={expiration}, timeLived={now - outputMessage.MessageInfo.SendTime}");
return;
}
if (outputMessage.State == OutputMessageState.Filtered || outputMessage.State == OutputMessageState.FilteredFailingDelivery)
{
if (outputMessage.RequestExpiration > now)
{
response.DeliveryOutcome = RequestOutcome.DELIVERING;
}
else
{
var deliverTask = this.DeliverAsync(taskInfo, now);
if (await Task.WhenAny(deliverTask, Task.Delay(PostTimeout)) == deliverTask)
{
response = deliverTask.Result;
}
else
{
response.DeliveryOutcome = RequestOutcome.TIMEOUT;
MessageDispatcherEventSource.Current.Warning(taskInfo.OutputMessage.MessageInfo.TrackingId, this, operation + ".Deliver", OperationStates.TimedOut, $"msgId={outputMessage.MessageInfo.MessageId}, batchId={outputMessage.Id}, outcome={response.DeliveryOutcome}, delivered={outputMessage.Delivered}");
}
}
}
if (response.DeliveryOutcome != RequestOutcome.FAILED_UNKNOWN)
{
this.resultReporter.ReporAndForgetAsync(outputMessage.ReportingServiceUri, new List <OutputResult> {
new OutputResult(outputMessage, response)
}.AsReadOnly());
}
// Only retry for unknown failure
else
{
if (outputMessage.DeliveryCount < this.setting.PartitionSetting.MaximumDeliveryCount)
{
outputMessage.DeliveryCount++;
}
else
{
MessageDispatcherEventSource.Current.Info(taskInfo.OutputMessage.MessageInfo.TrackingId, this, operation + ".Postpone", OperationStates.Dropped, $"msgId={outputMessage.MessageInfo.MessageId}, batchId={outputMessage.Id}, state={outputMessage.State}, deliveryCount={outputMessage.DeliveryCount}, outcome={response.DeliveryOutcome}, delivered={outputMessage.Delivered}, requestExpiration={outputMessage.RequestExpiration}, SendTime={outputMessage.MessageInfo.SendTime}, expired={expired}, expiration={expiration}, timeLived={now - outputMessage.MessageInfo.SendTime}");
this.resultReporter.ReporAndForgetAsync(outputMessage.ReportingServiceUri, new List <OutputResult> {
new OutputResult(outputMessage, response)
}.AsReadOnly());
return;
}
outputMessage.DeliveryTime = DateTime.UtcNow.Add(this.outputMessageQueue.Setting.RetryDelay);
TaskHelper.FireAndForget(() => this.messageDispatcher.PostponedAsync(new List <PushTaskInfo> {
taskInfo
}), ex => UnhandledException(ex, nameof(this.StartDeliveryTask)));
}
}
catch (Exception ex)
{
MessageDispatcherEventSource.Current.Warning(taskInfo.OutputMessage.MessageInfo.TrackingId, this, operation, OperationStates.Failed, ex.ToString());
taskInfo.Tcs.TrySetException(ex);
}
finally
{
taskInfo.Tcs.TrySetResult(true);
}
}