private void SendToAllWorkers(Message message, string logMessage)
{
var values = KnownWorkers.GetValues();
foreach (var worker in values)
{
var workerEndpoint = endpointRouter.GetRoutedEndpoint(worker);
using (var workerQueue = MsmqUtil.GetQueuePath(workerEndpoint).Open(QueueAccessMode.Send))
{
logger.DebugFormat(logMessage, Endpoint.Uri, worker);
workerQueue.Send(message);
}
}
if (values.Length == 0)
{
return;
}
var copy = MessageBatchSentToAllWorkers;
if (copy != null)
{
copy(message);
}
}
private void HandleReadyForWork(OpenedQueue queue, ReadyToWork work)
{
logger.DebugFormat("{0} is ready to work", work.Endpoint);
var needToAddToQueue = KnownWorkers.Add(work.Endpoint);
if (needToAddToQueue)
{
AddWorkerToQueue(queue, work);
}
readyForWork.Enqueue(work.Endpoint);
}
private void SendKnownWorkersAndKnownEndpoints(MessageQueue responseQueue)
{
if (responseQueue == null)
{
return;
}
try
{
var endpoints = KnownEndpoints.GetValues();
var workers = KnownWorkers.GetValues();
var transactionType = MessageQueueTransactionType.None;
if (Endpoint.Transactional.GetValueOrDefault())
{
transactionType = Transaction.Current == null ? MessageQueueTransactionType.Single : MessageQueueTransactionType.Automatic;
}
var index = 0;
while (index < endpoints.Length)
{
var endpointsBatch = endpoints
.Skip(index)
.Take(256)
.Select(x => new NewEndpointPersisted {
PersistedEndpoint = x
})
.ToArray();
index += endpointsBatch.Length;
responseQueue.Send(GenerateMsmqMessageFromMessageBatch(endpointsBatch), transactionType);
}
index = 0;
while (index < workers.Length)
{
var workersBatch = workers
.Skip(index)
.Take(256)
.Select(x => new NewWorkerPersisted {
Endpoint = x
})
.ToArray();
index += workersBatch.Length;
responseQueue.Send(GenerateMsmqMessageFromMessageBatch(workersBatch), transactionType);
}
}
catch (Exception e)
{
logger.Error("Failed to send known endpoints and known workers", e);
}
}
protected override void HandleLoadBalancerMessages(object msg)
{
var heartbeat = msg as Heartbeat;
if (heartbeat != null)
{
timeout.SetHeartbeat(DateTime.Now);
logger.Debug("Got heartbeat from primary");
if (tookOverWork)
{
logger.Info("Primary is now back in action, hurray!");
StartTrackingHeartbeats();
}
}
var readyForWorkQueueUriMessage = msg as ReadyForWorkQueueUri;
if (readyForWorkQueueUriMessage != null)
{
logger.InfoFormat("Got ReadyForWork endpoint from primary : {0}", readyForWorkQueueUriMessage.Endpoint);
readyForWorkQueueUriFromPrimary = readyForWorkQueueUriMessage.Endpoint;
}
var newEndpoint = msg as NewEndpointPersisted;
if (newEndpoint != null)
{
logger.InfoFormat("Got new endpoint persisted event from primary: {0}", newEndpoint.PersistedEndpoint);
KnownEndpoints.Add(newEndpoint.PersistedEndpoint);
}
var newWorker = msg as NewWorkerPersisted;
if (newWorker != null)
{
logger.InfoFormat("Got new worker persisted event from primary: {0}", newWorker.Endpoint);
KnownWorkers.Add(newWorker.Endpoint);
}
}
private void OnCheckPrimaryHeartbeat(object state)
{
lock (locker)
{
if (tookOverWork)
{
return;
}
if (TransportState != TransportState.Started)
{
return;
}
if (timeout.CheckTimestamp() == false)
{
return;
}
tookOverWork = true;
}
checkPrimaryHearteat.Dispose();
checkPrimaryHearteat = null;
foreach (var queueUri in KnownEndpoints.GetValues().Except(KnownWorkers.GetValues()))
{
if (queueUri == primaryLoadBalancer)
{
continue;
}
logger.InfoFormat("Notifying endpoints {0} that secondary load balancer {1} is taking over from {2}",
queueUri,
Endpoint.Uri,
PrimaryLoadBalancer
);
SendToQueue(queueUri, new Reroute
{
NewEndPoint = Endpoint.Uri,
OriginalEndPoint = PrimaryLoadBalancer
});
}
var newEndpoint = ReadyForWorkListener != null ? ReadyForWorkListener.Endpoint.Uri : Endpoint.Uri;
var originalEndPoint = readyForWorkQueueUriFromPrimary ?? primaryLoadBalancer;
foreach (var queueUri in KnownWorkers.GetValues())
{
logger.InfoFormat("Notifying worker {0} that secondary load balancer {1} is accepting work on awating listenerQueue",
queueUri,
newEndpoint,
originalEndPoint
);
SendToQueue(queueUri,
new Reroute
{
NewEndPoint = newEndpoint,
OriginalEndPoint = originalEndPoint
});
}
foreach (var queueUri in KnownWorkers.GetValues())
{
logger.InfoFormat("Notifying worker {0} that secondary load balancer {1} is accepting work",
queueUri,
Endpoint.Uri,
PrimaryLoadBalancer
);
SendToQueue(queueUri,
new AcceptingWork
{
Endpoint = Endpoint.Uri
});
}
Raise(TookOverAsActiveLoadBalancer);
}