private static void SyncBulkHandleMessages(NodeWithMessagesCollection distributedMessages, List <NodeWithMessages> unhandledNodes)
{
var waiters = new List <Action>();
for (int i = 0; i < distributedMessages.Count; i++)
{
NodeWithMessages nodeWithMessages = distributedMessages[i];
NodeWithInfo nodeWithInfo = nodeWithMessages.NodeWithInfo;
if (nodeWithMessages.Messages.InMessageCount > 0)
{
bool inMessagesHandled = nodeWithInfo.Node.HandleInMessages(nodeWithMessages.Messages.InMessages, nodeWithInfo.SyncInMessages, nodeWithInfo.SkipErrorQueueForSync);
if (!inMessagesHandled)
{
unhandledNodes.Add(nodeWithMessages);
}
}
if (nodeWithMessages.Messages.OutMessageCount > 0)
{
var node = nodeWithInfo.Node;
var ar = node.BeginHandleOutMessages(nodeWithMessages.Messages.OutMessages, null, null);
waiters.Add(() => node.EndHandleOutMessages(ar));
}
}
foreach (var waiter in waiters)
{
waiter();
}
}
/// <summary>
/// Performs processing on a block of messages
/// </summary>
/// <param name="messages">A list of RealyMessage objects to be processed</param>
/// <exception cref="SyncRelayOperationException">
/// When the type of an object is defined with settings
/// <see cref="MySpace.DataRelay.Common.Schemas.TypeSettings"></see> with
/// SyncInMessages=true and
/// ThrowOnSyncFailure=true
/// failed executions will throw this exception
/// </exception>
/// <remarks>
/// Procssing steps:
/// 1. Splits the list of messages into several lists, each containing
/// the node, messages of a single type and settings for that type
/// 2. Processes each list individually
/// 3. Tracks the success/failure of each list
/// 4. If a list (or a message from a list) fails, and the type of message
/// requires a "Throw" when sync fails: throws a SyncRelayOperationException,
/// otherwise, places failed messages into the exception queue
/// </remarks>
public virtual void HandleMessages(IList <RelayMessage> messages)
{
SetHydrationPolicy(messages);
NodeManager.Instance.Counters.CountMessageList(messages);
NodeWithMessagesCollection distributedMessages = NodeManager.Instance.DistributeMessages(messages);
List <NodeWithMessages> unhandledNodes = new List <NodeWithMessages>();
if (_enableAsyncBulkGets && distributedMessages.Count > 1) // only do the async if there's more than 1 node to send to. No point in hitting a wait handle if we don't need to coordinate a response
{
while (distributedMessages.Count > 0) //keep trying until retries have been exhausted
{
AsyncBulkHandleMessages(distributedMessages, unhandledNodes);
distributedMessages = NodeManager.Instance.RedistributeRetryMessages(distributedMessages);
}
}
else
{
while (distributedMessages.Count > 0)
{
SyncBulkHandleMessages(distributedMessages, unhandledNodes);
distributedMessages = NodeManager.Instance.RedistributeRetryMessages(distributedMessages);
}
}
if (unhandledNodes.Count > 0)
{
bool bThrow = false;
StringBuilder detailBuilder = new StringBuilder();
detailBuilder.Append("Error handling sync in messages\r\n");
foreach (NodeWithMessages nwm in unhandledNodes)
{
if (nwm.NodeWithInfo.SkipErrorQueueForSync && nwm.NodeWithInfo.SyncInMessages)
{
detailBuilder.AppendFormat("Node:{0}\r\n", nwm.NodeWithInfo.Node.GetType().Name);
detailBuilder.AppendFormat("\tMessages: {0}\r\n", nwm.Messages.InMessageCount);
bThrow = true;
}
}
if (bThrow)
{
throw new SyncRelayOperationException(detailBuilder.ToString());
}
if (log.IsInfoEnabled)
{
log.Info(detailBuilder.ToString());
}
}
}
/// <summary>
/// Create a list of nodes with messages that can be retried according to Relay Group settings.
/// </summary>
/// <param name="distributedMessages">A list of distributed messages have been previously attempted.</param>
/// <returns>A redistribution of nodes with messages left to retry, or null if no retries are needed or allowed.</returns>
internal NodeWithMessagesCollection RedistributeRetryMessages(NodeWithMessagesCollection distributedMessages)
{
var redistributedMessages = new NodeWithMessagesCollection();
foreach (var nodeWithMessages in distributedMessages)
{
var retriesAllowed = nodeWithMessages.NodeWithInfo.Node.NodeGroup.GroupDefinition.RetryCount;
var retriesAttempted = nodeWithMessages.AttemptedNodes.Count;
if (nodeWithMessages.Messages.OutMessageCount > 0 && retriesAttempted < retriesAllowed)
{
var firstMessage = nodeWithMessages.Messages.OutMessages[0];
var retryable = firstMessage.IsRetryable(Instance.GetRelayRetryPolicyForMessage(firstMessage));
// the first message's outcome should be the same as every message in the list for these specific conditions
if (retryable)
{
nodeWithMessages.AttemptedNodes.Add(nodeWithMessages.NodeWithInfo.Node);
var retryNode =
nodeWithMessages.NodeWithInfo.Node.GetRetryNodeFromCluster(nodeWithMessages.AttemptedNodes);
if (retryNode != null)
{
// only try to redistribute this node if a retryNode is available. Otherwise, retries are not possible.
nodeWithMessages.NodeWithInfo.Node = retryNode;
// prepare each message in the list for retry.
foreach (var message in nodeWithMessages.Messages.OutMessages)
{
message.RelayTTL++;
message.SetError(RelayErrorType.None);
message.ResultOutcome = RelayOutcome.NotSent;
}
// wipe out any potential IN messages.
nodeWithMessages.Messages.InMessages = null;
redistributedMessages.Add(nodeWithMessages);
}
}
}
}
return(redistributedMessages);
}
/// <summary>
/// Performs processing on a block of messages
/// </summary>
/// <param name="messages">A list of RealyMessage objects to be processed</param>
/// <exception cref="SyncRelayOperationException">
/// When the type of an object is defined with settings
/// <see cref="MySpace.DataRelay.Common.Schemas.TypeSettings"></see> with
/// SyncInMessages=true and
/// ThrowOnSyncFailure=true
/// failed executions will throw this exception
/// </exception>
/// <remarks>
/// Procssing steps:
/// 1. Splits the list of messages into several lists, each containing
/// the node, messages of a single type and settings for that type
/// 2. Processes each list individually
/// 3. Tracks the success/failure of each list
/// 4. If a list (or a message from a list) fails, and the type of message
/// requires a "Throw" when sync fails: throws a SyncRelayOperationException,
/// otherwise, places failed messages into the exception queue
/// </remarks>
public void HandleMessages(IList <RelayMessage> messages)
{
SetHydrationPolicy(messages);
NodeManager.Instance.Counters.CountMessageList(messages);
NodeWithMessagesCollection distributedMessages = NodeManager.Instance.DistributeMessages(messages);
NodeWithMessages nodeMessages;
AutoResetEvent resetEvent = null;
HandleWithCount finishedLock = null;
List <NodeWithMessages> unhandledNodes = new List <NodeWithMessages>();
if (_enableAsyncBulkGets && distributedMessages.Count > 1) //only do the async if there's more than 1 node to send to. No point in hitting a wait handle if we don't need to coordinate a response
{
int numberOfGets = 0, getsLefts = 0;
for (int i = 0; i < distributedMessages.Count; i++)
{
if (distributedMessages[i].Messages.OutMessageCount > 0)
{
numberOfGets++;
}
}
if (numberOfGets > 0)
{
resetEvent = OutMessageWaitHandle;
finishedLock = new HandleWithCount(resetEvent, numberOfGets);
getsLefts = numberOfGets; //use this to determine when we're at the last get so we use this thread to process it
}
for (int i = 0; i < distributedMessages.Count; i++)
{
nodeMessages = distributedMessages[i];
if (nodeMessages.Messages.InMessageCount > 0)
{
bool inMessagesHandled = nodeMessages.NodeWithInfo.Node.HandleInMessages(nodeMessages.Messages.InMessages,
nodeMessages.NodeWithInfo.SyncInMessages, nodeMessages.NodeWithInfo.SkipErrorQueueForSync);
if (!inMessagesHandled)
{
unhandledNodes.Add(nodeMessages);
}
}
if (nodeMessages.Messages.OutMessageCount > 0)
{
if (--getsLefts > 0)
{
nodeMessages.NodeWithInfo.Node.PostOutMessages(
new MessagesWithLock(nodeMessages.Messages.OutMessages, finishedLock));
}
else
{
nodeMessages.NodeWithInfo.Node.HandleOutMessages(nodeMessages.Messages.OutMessages);
finishedLock.Decrement();
}
}
}
if (numberOfGets > 0)
{
resetEvent.WaitOne();
}
return;
}
else
{
for (int i = 0; i < distributedMessages.Count; i++)
{
nodeMessages = distributedMessages[i];
if (nodeMessages.Messages.InMessageCount > 0)
{
bool inMessagesHandled = nodeMessages.NodeWithInfo.Node.HandleInMessages(nodeMessages.Messages.InMessages, nodeMessages.NodeWithInfo.SyncInMessages, nodeMessages.NodeWithInfo.SkipErrorQueueForSync);
if (!inMessagesHandled)
{
unhandledNodes.Add(nodeMessages);
}
}
if (nodeMessages.Messages.OutMessageCount > 0)
{
nodeMessages.NodeWithInfo.Node.HandleOutMessages(nodeMessages.Messages.OutMessages);
}
}
}
if (unhandledNodes.Count > 0)
{
bool bThrow = false;
StringBuilder detailBuilder = new StringBuilder();
detailBuilder.Append("Error handling sync in messages\r\n");
foreach (NodeWithMessages nwm in unhandledNodes)
{
if (nwm.NodeWithInfo.SkipErrorQueueForSync && nwm.NodeWithInfo.SyncInMessages)
{
detailBuilder.AppendFormat("Node:{0}\r\n", nwm.NodeWithInfo.Node.GetType().Name);
detailBuilder.AppendFormat("\tMessages: {0}\r\n", nwm.Messages.InMessageCount);
bThrow = true;
}
}
if (bThrow)
{
throw new SyncRelayOperationException(detailBuilder.ToString());
}
if (_log.IsInfoEnabled)
{
_log.Info(detailBuilder.ToString());
}
}
}
private static void AsyncBulkHandleMessages(NodeWithMessagesCollection distributedMessages, List <NodeWithMessages> unhandledNodes)
{
if (log.IsDebugEnabled)
{
log.Debug("Starting asyncbulk handle messages");
}
AutoResetEvent resetEvent = null;
HandleWithCount finishedLock = null;
int numberOfGets = 0;
int getsLefts = 0;
for (int i = 0; i < distributedMessages.Count; i++)
{
if (distributedMessages[i].Messages.OutMessageCount > 0)
{
numberOfGets++;
}
}
if (numberOfGets > 0)
{
resetEvent = OutMessageWaitHandle;
finishedLock = new HandleWithCount(resetEvent, numberOfGets);
getsLefts = numberOfGets; //use this to determine when we're at the last get so we use this thread to process it
}
for (int i = 0; i < distributedMessages.Count; i++)
{
NodeWithMessages nodeWithMessages = distributedMessages[i];
NodeWithInfo nodeWithInfo = nodeWithMessages.NodeWithInfo;
if (nodeWithMessages.Messages.InMessageCount > 0)
{
bool inMessagesHandled = nodeWithInfo.Node.HandleInMessages(nodeWithMessages.Messages.InMessages,
nodeWithInfo.SyncInMessages, nodeWithInfo.SkipErrorQueueForSync);
if (!inMessagesHandled)
{
unhandledNodes.Add(nodeWithMessages);
}
}
if (nodeWithMessages.Messages.OutMessageCount > 0)
{
if (--getsLefts > 0) //post all the last group to be handled async
{
nodeWithInfo.Node.PostOutMessages(
new MessagesWithLock(nodeWithMessages.Messages.OutMessages, finishedLock));
}
else
{
nodeWithInfo.Node.SendOutMessages(nodeWithMessages.Messages.OutMessages); //do the last set on this thread rather than punting the duty
finishedLock.Decrement();
}
}
}
if (numberOfGets > 0)
{
resetEvent.WaitOne(); //wait for all operations to complete. the event will be signaled with lock.decrement hits 0
}
}
/// <summary>
/// Splits messages into various lists of in and out message destined for different nodes.
/// </summary>
/// <param name="messages"></param>
/// <returns></returns>
internal NodeWithMessagesCollection DistributeMessages(IList <RelayMessage> messages)
{
NodeWithMessagesCollection distribution = new NodeWithMessagesCollection();
RelayMessage message;
Node node;
for (int i = 0; i < messages.Count; i++)
{
if (messages[i] != null)
{
message = messages[i];
RelayMessage interZoneMessage = null;
SimpleLinkedList <Node> nodesForMessage = GetNodesForMessage(message);
SimpleLinkedList <Node> nodesForInterZoneMessage = null;
if (nodesForMessage.Count > 0)
{
message.AddressHistory.Add(MyIpAddress);
}
message.RelayTTL--;
#region Identify InterZone Messages
if (message.IsTwoWayMessage == false)
{
message.ResultOutcome = RelayOutcome.Queued; //will be queued, if sync will not get overwritten
// Identify nodes in foreign zones
int nodeCount = nodesForMessage.Count;
for (int nodeIndex = 0; nodeIndex < nodeCount; nodeIndex++)
{
nodesForMessage.Pop(out node);
if (_myNodeDefinition != null && _myNodeDefinition.Zone != node.NodeDefinition.Zone)
{
// Message needs to cross Zone bounderies
if (interZoneMessage == null)
{
interZoneMessage = RelayMessage.CreateInterZoneMessageFrom(message);
nodesForInterZoneMessage = new SimpleLinkedList <Node>();
}
nodesForInterZoneMessage.Push(node);
}
else
{
nodesForMessage.Push(node);
}
}
}
#endregion
if (nodesForMessage.Count > 0)
{
DebugWriter.WriteDebugInfo(message, nodesForMessage);
distribution.Add(message, nodesForMessage);
}
if (nodesForInterZoneMessage != null && nodesForInterZoneMessage.Count > 0)
{
DebugWriter.WriteDebugInfo(interZoneMessage, nodesForInterZoneMessage);
distribution.Add(interZoneMessage, nodesForInterZoneMessage);
}
}
}
return(distribution);
}
