private static void SyncBulkHandleMessages(NodeWithMessagesCollection distributedMessages, List <NodeWithMessages> unhandledNodes) { var waiters = new List <Action>(); for (int i = 0; i < distributedMessages.Count; i++) { NodeWithMessages nodeWithMessages = distributedMessages[i]; NodeWithInfo nodeWithInfo = nodeWithMessages.NodeWithInfo; if (nodeWithMessages.Messages.InMessageCount > 0) { bool inMessagesHandled = nodeWithInfo.Node.HandleInMessages(nodeWithMessages.Messages.InMessages, nodeWithInfo.SyncInMessages, nodeWithInfo.SkipErrorQueueForSync); if (!inMessagesHandled) { unhandledNodes.Add(nodeWithMessages); } } if (nodeWithMessages.Messages.OutMessageCount > 0) { var node = nodeWithInfo.Node; var ar = node.BeginHandleOutMessages(nodeWithMessages.Messages.OutMessages, null, null); waiters.Add(() => node.EndHandleOutMessages(ar)); } } foreach (var waiter in waiters) { waiter(); } }
/// <summary> /// Performs processing on a block of messages /// </summary> /// <param name="messages">A list of RealyMessage objects to be processed</param> /// <exception cref="SyncRelayOperationException"> /// When the type of an object is defined with settings /// <see cref="MySpace.DataRelay.Common.Schemas.TypeSettings"></see> with /// SyncInMessages=true and /// ThrowOnSyncFailure=true /// failed executions will throw this exception /// </exception> /// <remarks> /// Procssing steps: /// 1. Splits the list of messages into several lists, each containing /// the node, messages of a single type and settings for that type /// 2. Processes each list individually /// 3. Tracks the success/failure of each list /// 4. If a list (or a message from a list) fails, and the type of message /// requires a "Throw" when sync fails: throws a SyncRelayOperationException, /// otherwise, places failed messages into the exception queue /// </remarks> public virtual void HandleMessages(IList <RelayMessage> messages) { SetHydrationPolicy(messages); NodeManager.Instance.Counters.CountMessageList(messages); NodeWithMessagesCollection distributedMessages = NodeManager.Instance.DistributeMessages(messages); List <NodeWithMessages> unhandledNodes = new List <NodeWithMessages>(); if (_enableAsyncBulkGets && distributedMessages.Count > 1) // only do the async if there's more than 1 node to send to. No point in hitting a wait handle if we don't need to coordinate a response { while (distributedMessages.Count > 0) //keep trying until retries have been exhausted { AsyncBulkHandleMessages(distributedMessages, unhandledNodes); distributedMessages = NodeManager.Instance.RedistributeRetryMessages(distributedMessages); } } else { while (distributedMessages.Count > 0) { SyncBulkHandleMessages(distributedMessages, unhandledNodes); distributedMessages = NodeManager.Instance.RedistributeRetryMessages(distributedMessages); } } if (unhandledNodes.Count > 0) { bool bThrow = false; StringBuilder detailBuilder = new StringBuilder(); detailBuilder.Append("Error handling sync in messages\r\n"); foreach (NodeWithMessages nwm in unhandledNodes) { if (nwm.NodeWithInfo.SkipErrorQueueForSync && nwm.NodeWithInfo.SyncInMessages) { detailBuilder.AppendFormat("Node:{0}\r\n", nwm.NodeWithInfo.Node.GetType().Name); detailBuilder.AppendFormat("\tMessages: {0}\r\n", nwm.Messages.InMessageCount); bThrow = true; } } if (bThrow) { throw new SyncRelayOperationException(detailBuilder.ToString()); } if (log.IsInfoEnabled) { log.Info(detailBuilder.ToString()); } } }
/// <summary> /// Create a list of nodes with messages that can be retried according to Relay Group settings. /// </summary> /// <param name="distributedMessages">A list of distributed messages have been previously attempted.</param> /// <returns>A redistribution of nodes with messages left to retry, or null if no retries are needed or allowed.</returns> internal NodeWithMessagesCollection RedistributeRetryMessages(NodeWithMessagesCollection distributedMessages) { var redistributedMessages = new NodeWithMessagesCollection(); foreach (var nodeWithMessages in distributedMessages) { var retriesAllowed = nodeWithMessages.NodeWithInfo.Node.NodeGroup.GroupDefinition.RetryCount; var retriesAttempted = nodeWithMessages.AttemptedNodes.Count; if (nodeWithMessages.Messages.OutMessageCount > 0 && retriesAttempted < retriesAllowed) { var firstMessage = nodeWithMessages.Messages.OutMessages[0]; var retryable = firstMessage.IsRetryable(Instance.GetRelayRetryPolicyForMessage(firstMessage)); // the first message's outcome should be the same as every message in the list for these specific conditions if (retryable) { nodeWithMessages.AttemptedNodes.Add(nodeWithMessages.NodeWithInfo.Node); var retryNode = nodeWithMessages.NodeWithInfo.Node.GetRetryNodeFromCluster(nodeWithMessages.AttemptedNodes); if (retryNode != null) { // only try to redistribute this node if a retryNode is available. Otherwise, retries are not possible. nodeWithMessages.NodeWithInfo.Node = retryNode; // prepare each message in the list for retry. foreach (var message in nodeWithMessages.Messages.OutMessages) { message.RelayTTL++; message.SetError(RelayErrorType.None); message.ResultOutcome = RelayOutcome.NotSent; } // wipe out any potential IN messages. nodeWithMessages.Messages.InMessages = null; redistributedMessages.Add(nodeWithMessages); } } } } return(redistributedMessages); }
/// <summary> /// Performs processing on a block of messages /// </summary> /// <param name="messages">A list of RealyMessage objects to be processed</param> /// <exception cref="SyncRelayOperationException"> /// When the type of an object is defined with settings /// <see cref="MySpace.DataRelay.Common.Schemas.TypeSettings"></see> with /// SyncInMessages=true and /// ThrowOnSyncFailure=true /// failed executions will throw this exception /// </exception> /// <remarks> /// Procssing steps: /// 1. Splits the list of messages into several lists, each containing /// the node, messages of a single type and settings for that type /// 2. Processes each list individually /// 3. Tracks the success/failure of each list /// 4. If a list (or a message from a list) fails, and the type of message /// requires a "Throw" when sync fails: throws a SyncRelayOperationException, /// otherwise, places failed messages into the exception queue /// </remarks> public void HandleMessages(IList <RelayMessage> messages) { SetHydrationPolicy(messages); NodeManager.Instance.Counters.CountMessageList(messages); NodeWithMessagesCollection distributedMessages = NodeManager.Instance.DistributeMessages(messages); NodeWithMessages nodeMessages; AutoResetEvent resetEvent = null; HandleWithCount finishedLock = null; List <NodeWithMessages> unhandledNodes = new List <NodeWithMessages>(); if (_enableAsyncBulkGets && distributedMessages.Count > 1) //only do the async if there's more than 1 node to send to. No point in hitting a wait handle if we don't need to coordinate a response { int numberOfGets = 0, getsLefts = 0; for (int i = 0; i < distributedMessages.Count; i++) { if (distributedMessages[i].Messages.OutMessageCount > 0) { numberOfGets++; } } if (numberOfGets > 0) { resetEvent = OutMessageWaitHandle; finishedLock = new HandleWithCount(resetEvent, numberOfGets); getsLefts = numberOfGets; //use this to determine when we're at the last get so we use this thread to process it } for (int i = 0; i < distributedMessages.Count; i++) { nodeMessages = distributedMessages[i]; if (nodeMessages.Messages.InMessageCount > 0) { bool inMessagesHandled = nodeMessages.NodeWithInfo.Node.HandleInMessages(nodeMessages.Messages.InMessages, nodeMessages.NodeWithInfo.SyncInMessages, nodeMessages.NodeWithInfo.SkipErrorQueueForSync); if (!inMessagesHandled) { unhandledNodes.Add(nodeMessages); } } if (nodeMessages.Messages.OutMessageCount > 0) { if (--getsLefts > 0) { nodeMessages.NodeWithInfo.Node.PostOutMessages( new MessagesWithLock(nodeMessages.Messages.OutMessages, finishedLock)); } else { nodeMessages.NodeWithInfo.Node.HandleOutMessages(nodeMessages.Messages.OutMessages); finishedLock.Decrement(); } } } if (numberOfGets > 0) { resetEvent.WaitOne(); } return; } else { for (int i = 0; i < distributedMessages.Count; i++) { nodeMessages = distributedMessages[i]; if (nodeMessages.Messages.InMessageCount > 0) { bool inMessagesHandled = nodeMessages.NodeWithInfo.Node.HandleInMessages(nodeMessages.Messages.InMessages, nodeMessages.NodeWithInfo.SyncInMessages, nodeMessages.NodeWithInfo.SkipErrorQueueForSync); if (!inMessagesHandled) { unhandledNodes.Add(nodeMessages); } } if (nodeMessages.Messages.OutMessageCount > 0) { nodeMessages.NodeWithInfo.Node.HandleOutMessages(nodeMessages.Messages.OutMessages); } } } if (unhandledNodes.Count > 0) { bool bThrow = false; StringBuilder detailBuilder = new StringBuilder(); detailBuilder.Append("Error handling sync in messages\r\n"); foreach (NodeWithMessages nwm in unhandledNodes) { if (nwm.NodeWithInfo.SkipErrorQueueForSync && nwm.NodeWithInfo.SyncInMessages) { detailBuilder.AppendFormat("Node:{0}\r\n", nwm.NodeWithInfo.Node.GetType().Name); detailBuilder.AppendFormat("\tMessages: {0}\r\n", nwm.Messages.InMessageCount); bThrow = true; } } if (bThrow) { throw new SyncRelayOperationException(detailBuilder.ToString()); } if (_log.IsInfoEnabled) { _log.Info(detailBuilder.ToString()); } } }
private static void AsyncBulkHandleMessages(NodeWithMessagesCollection distributedMessages, List <NodeWithMessages> unhandledNodes) { if (log.IsDebugEnabled) { log.Debug("Starting asyncbulk handle messages"); } AutoResetEvent resetEvent = null; HandleWithCount finishedLock = null; int numberOfGets = 0; int getsLefts = 0; for (int i = 0; i < distributedMessages.Count; i++) { if (distributedMessages[i].Messages.OutMessageCount > 0) { numberOfGets++; } } if (numberOfGets > 0) { resetEvent = OutMessageWaitHandle; finishedLock = new HandleWithCount(resetEvent, numberOfGets); getsLefts = numberOfGets; //use this to determine when we're at the last get so we use this thread to process it } for (int i = 0; i < distributedMessages.Count; i++) { NodeWithMessages nodeWithMessages = distributedMessages[i]; NodeWithInfo nodeWithInfo = nodeWithMessages.NodeWithInfo; if (nodeWithMessages.Messages.InMessageCount > 0) { bool inMessagesHandled = nodeWithInfo.Node.HandleInMessages(nodeWithMessages.Messages.InMessages, nodeWithInfo.SyncInMessages, nodeWithInfo.SkipErrorQueueForSync); if (!inMessagesHandled) { unhandledNodes.Add(nodeWithMessages); } } if (nodeWithMessages.Messages.OutMessageCount > 0) { if (--getsLefts > 0) //post all the last group to be handled async { nodeWithInfo.Node.PostOutMessages( new MessagesWithLock(nodeWithMessages.Messages.OutMessages, finishedLock)); } else { nodeWithInfo.Node.SendOutMessages(nodeWithMessages.Messages.OutMessages); //do the last set on this thread rather than punting the duty finishedLock.Decrement(); } } } if (numberOfGets > 0) { resetEvent.WaitOne(); //wait for all operations to complete. the event will be signaled with lock.decrement hits 0 } }
/// <summary> /// Splits messages into various lists of in and out message destined for different nodes. /// </summary> /// <param name="messages"></param> /// <returns></returns> internal NodeWithMessagesCollection DistributeMessages(IList <RelayMessage> messages) { NodeWithMessagesCollection distribution = new NodeWithMessagesCollection(); RelayMessage message; Node node; for (int i = 0; i < messages.Count; i++) { if (messages[i] != null) { message = messages[i]; RelayMessage interZoneMessage = null; SimpleLinkedList <Node> nodesForMessage = GetNodesForMessage(message); SimpleLinkedList <Node> nodesForInterZoneMessage = null; if (nodesForMessage.Count > 0) { message.AddressHistory.Add(MyIpAddress); } message.RelayTTL--; #region Identify InterZone Messages if (message.IsTwoWayMessage == false) { message.ResultOutcome = RelayOutcome.Queued; //will be queued, if sync will not get overwritten // Identify nodes in foreign zones int nodeCount = nodesForMessage.Count; for (int nodeIndex = 0; nodeIndex < nodeCount; nodeIndex++) { nodesForMessage.Pop(out node); if (_myNodeDefinition != null && _myNodeDefinition.Zone != node.NodeDefinition.Zone) { // Message needs to cross Zone bounderies if (interZoneMessage == null) { interZoneMessage = RelayMessage.CreateInterZoneMessageFrom(message); nodesForInterZoneMessage = new SimpleLinkedList <Node>(); } nodesForInterZoneMessage.Push(node); } else { nodesForMessage.Push(node); } } } #endregion if (nodesForMessage.Count > 0) { DebugWriter.WriteDebugInfo(message, nodesForMessage); distribution.Add(message, nodesForMessage); } if (nodesForInterZoneMessage != null && nodesForInterZoneMessage.Count > 0) { DebugWriter.WriteDebugInfo(interZoneMessage, nodesForInterZoneMessage); distribution.Add(interZoneMessage, nodesForInterZoneMessage); } } } return(distribution); }