protected override void AfterStart(OpenedQueue queue)
{
if (_readyForWorkListener != null)
{
_readyForWorkListener.ReadyToWorkMessageArrived += readyForWorkMessage => HandleReadyForWork(queue, readyForWorkMessage);
_readyForWorkListener.Start();
}
if (secondaryLoadBalancer != null)
{
foreach (var queueUri in KnownEndpoints.GetValues())
{
logger.InfoFormat("Notifying {0} that primary load balancer {1} is taking over from secondary",
queueUri,
Endpoint.Uri
);
SendToQueue(queueUri, new Reroute
{
NewEndPoint = Endpoint.Uri,
OriginalEndPoint = Endpoint.Uri
});
}
SendHeartBeatToSecondaryServer(null);
heartBeatTimer.Change(TimeSpan.FromSeconds(1), TimeSpan.FromSeconds(1));
Reroute reroute;
if (_readyForWorkListener != null)
{
reroute = new Reroute
{
NewEndPoint = _readyForWorkListener.Endpoint.Uri,
OriginalEndPoint = _readyForWorkListener.Endpoint.Uri
}
}
;
else
{
reroute = new Reroute
{
NewEndPoint = Endpoint.Uri,
OriginalEndPoint = Endpoint.Uri
}
};
SendToAllWorkers(
GenerateMsmqMessageFromMessageBatch(reroute),
"Rerouting {1} back to {0}"
);
}
if (ShouldNotifyWorkersLoaderIsReadyToAcceptWorkOnStartup)
{
NotifyWorkersThatLoaderIsReadyToAcceptWork();
}
}
private void SendKnownWorkersAndKnownEndpoints(MessageQueue responseQueue)
{
if (responseQueue == null)
{
return;
}
try
{
var endpoints = KnownEndpoints.GetValues();
var workers = KnownWorkers.GetValues();
var transactionType = MessageQueueTransactionType.None;
if (Endpoint.Transactional.GetValueOrDefault())
{
transactionType = Transaction.Current == null ? MessageQueueTransactionType.Single : MessageQueueTransactionType.Automatic;
}
var index = 0;
while (index < endpoints.Length)
{
var endpointsBatch = endpoints
.Skip(index)
.Take(256)
.Select(x => new NewEndpointPersisted {
PersistedEndpoint = x
})
.ToArray();
index += endpointsBatch.Length;
responseQueue.Send(GenerateMsmqMessageFromMessageBatch(endpointsBatch), transactionType);
}
index = 0;
while (index < workers.Length)
{
var workersBatch = workers
.Skip(index)
.Take(256)
.Select(x => new NewWorkerPersisted {
Endpoint = x
})
.ToArray();
index += workersBatch.Length;
responseQueue.Send(GenerateMsmqMessageFromMessageBatch(workersBatch), transactionType);
}
}
catch (Exception e)
{
logger.Error("Failed to send known endpoints and known workers", e);
}
}
protected override void HandleLoadBalancerMessages(object msg)
{
var heartbeat = msg as Heartbeat;
if (heartbeat != null)
{
timeout.SetHeartbeat(DateTime.Now);
logger.Debug("Got heartbeat from primary");
if (tookOverWork)
{
logger.Info("Primary is now back in action, hurray!");
StartTrackingHeartbeats();
}
}
var readyForWorkQueueUriMessage = msg as ReadyForWorkQueueUri;
if (readyForWorkQueueUriMessage != null)
{
logger.InfoFormat("Got ReadyForWork endpoint from primary : {0}", readyForWorkQueueUriMessage.Endpoint);
readyForWorkQueueUriFromPrimary = readyForWorkQueueUriMessage.Endpoint;
}
var newEndpoint = msg as NewEndpointPersisted;
if (newEndpoint != null)
{
logger.InfoFormat("Got new endpoint persisted event from primary: {0}", newEndpoint.PersistedEndpoint);
KnownEndpoints.Add(newEndpoint.PersistedEndpoint);
}
var newWorker = msg as NewWorkerPersisted;
if (newWorker != null)
{
logger.InfoFormat("Got new worker persisted event from primary: {0}", newWorker.Endpoint);
KnownWorkers.Add(newWorker.Endpoint);
}
}
private void OnCheckPrimaryHeartbeat(object state)
{
lock (locker)
{
if (tookOverWork)
{
return;
}
if (TransportState != TransportState.Started)
{
return;
}
if (timeout.CheckTimestamp() == false)
{
return;
}
tookOverWork = true;
}
checkPrimaryHearteat.Dispose();
checkPrimaryHearteat = null;
foreach (var queueUri in KnownEndpoints.GetValues().Except(KnownWorkers.GetValues()))
{
if (queueUri == primaryLoadBalancer)
{
continue;
}
logger.InfoFormat("Notifying endpoints {0} that secondary load balancer {1} is taking over from {2}",
queueUri,
Endpoint.Uri,
PrimaryLoadBalancer
);
SendToQueue(queueUri, new Reroute
{
NewEndPoint = Endpoint.Uri,
OriginalEndPoint = PrimaryLoadBalancer
});
}
var newEndpoint = ReadyForWorkListener != null ? ReadyForWorkListener.Endpoint.Uri : Endpoint.Uri;
var originalEndPoint = readyForWorkQueueUriFromPrimary ?? primaryLoadBalancer;
foreach (var queueUri in KnownWorkers.GetValues())
{
logger.InfoFormat("Notifying worker {0} that secondary load balancer {1} is accepting work on awating listenerQueue",
queueUri,
newEndpoint,
originalEndPoint
);
SendToQueue(queueUri,
new Reroute
{
NewEndPoint = newEndpoint,
OriginalEndPoint = originalEndPoint
});
}
foreach (var queueUri in KnownWorkers.GetValues())
{
logger.InfoFormat("Notifying worker {0} that secondary load balancer {1} is accepting work",
queueUri,
Endpoint.Uri,
PrimaryLoadBalancer
);
SendToQueue(queueUri,
new AcceptingWork
{
Endpoint = Endpoint.Uri
});
}
Raise(TookOverAsActiveLoadBalancer);
}
