protected override async Task Send(List <KeyValuePair <long, IMessage> > toSend)
{
// serialize payload
MemoryStream stream = new MemoryStream();
using (var binaryDictionaryWriter = XmlDictionaryWriter.CreateBinaryWriter(stream))
{
payloadSerializer.WriteObject(binaryDictionaryWriter, toSend);
stream.Flush();
}
var now = DateTime.UtcNow;
// send message
var message = new ProcessMessage()
{
DeploymentTimestamp = deploymentTimestamp,
Source = processId,
LastClock = toSend[toSend.Count - 1].Key,
Payload = stream.ToArray()
};
var messageBytes = ProcessMessage.Serialize(message);
await sender.SendAsync(new EventData(messageBytes));
logger.LogTrace($"{DateTime.UtcNow:o} Sent {message}->{destination} ({messageBytes.Length / 1024}kB)");
if (lastSendOrDoorbell + configuration.ReceiveWaitTime < now)
{
// send doorbell message
var doorbellMessage = new DoorbellMessage()
{
ProcessId = destination,
Guid = Guid.NewGuid()
};
var doorbell = connections.GetDoorbellSender(destination);
await doorbell.SendAsync(new EventData(DoorbellMessage.Serialize(doorbellMessage)));
logger.LogTrace($"{DateTime.UtcNow:o} Sent {doorbellMessage}");
}
lastSendOrDoorbell = now;
}
public async Task FinalRecheck()
{
var eventData = await Connections.ProcessReceiver.ReceiveAsync(1, TimeSpan.FromMilliseconds(1));
if (eventData != null)
{
var eventEnumerator = eventData.GetEnumerator();
if (eventEnumerator.MoveNext())
{
// there was a race... we missed a message
var msg = ProcessMessage.Deserialize(eventEnumerator.Current.Body.Array);
// send a ping to ourselves before shutting down
// so we will wake up and resume
HostLogger.LogDebug($"{DateTime.UtcNow:o} detected a new message {msg}->{processId} at position {eventEnumerator.Current.SystemProperties.SequenceNumber} after already releasing lease - ring my own doorbell");
await RingMyself();
}
}
}
public async Task <bool> ResumeFromCheckpoint(LeaseManager leaseManager)
{
var start = stopwatch.Elapsed;
try
{
// kick off the load
var storageConnectionString = configuration.StorageConnectionString;
var loadtask = AzureBlobStorageStateManager.Load(storageConnectionString, HostLogger, processId);
// register host services
application.HostServices.RegisterSend(Send);
application.HostServices.RegisterApplicationLogger(ApplicationLogger);
application.HostServices.RegisterRuntimeLogger(RuntimeLogger);
// if (_configuration.AppInsightsInstrumentationKey != null)
// _application.HostServices.RegisterTelemetryListener(new ApplicationInsightsTelemetryListener(_configuration.AppInsightsInstrumentationKey, "eventhubs"));
if (blobTelemetryListener != null)
{
application.HostServices.RegisterTelemetryListener(blobTelemetryListener);
}
// read the checkpoint
var checkpointedState = await loadtask;
SetDeploymentTimestamp(checkpointedState.DeploymentTimestamp);
seenClock = checkpointedState.SeenClock;
ourClock = checkpointedState.OurClock;
var state = checkpointedState.State;
lastCheckpointedSequenceNumber = checkpointedState.Version;
HostLogger.LogDebug($"Resuming at position {lastCheckpointedSequenceNumber}");
Connections.ResumeFrom(lastCheckpointedSequenceNumber);
// build the process
IProcess process = application.MakeProcess(processId);
if (state == null)
{
process.FirstStart();
}
else
{
process.Restore(state, out var label);
}
process.BecomePrimary();
// start the message receiving loop
long lastProcessedPosition = 0;
int iterationCount = 0;
int dedupCount = 0;
int receiveCount = 0;
int loopbackCount = 0;
int clientCount = 0;
TimeSpan lastReport = stopwatch.Elapsed;
long lastReportedPosition = 0;
while (true)
{
iterationCount++;
if (lastProcessedPosition > lastReportedPosition && stopwatch.Elapsed - lastReport > TimeSpan.FromSeconds(15))
{
HostLogger.LogInformation($"progress to v{lastProcessedPosition} after {stopwatch.Elapsed.TotalSeconds:f2}s, {receiveCount + dedupCount + loopbackCount + clientCount} messages ({receiveCount} new, {loopbackCount} loopback, {clientCount} client, {dedupCount} deduped) in {iterationCount} batches");
lastReportedPosition = lastProcessedPosition;
lastReport = stopwatch.Elapsed;
CombinedLogger.Flush();
}
try
{
bool outOfTime = stopwatch.Elapsed > configuration.TimeLimit;
IEnumerable <EventData> eventData = outOfTime ? null :
await Connections.ProcessReceiver.ReceiveAsync(configuration.MaxReceiveBatchSize, iterationCount == 1?TimeSpan.FromSeconds(10) : configuration.ReceiveWaitTime);
if (eventData == null)
{
HostLogger.LogTrace($"{ DateTime.UtcNow:o} Received nothing. {Connections.ProcessReceiver.RuntimeInfo.LastSequenceNumber}");
if (process.RequestsOutstanding() && !outOfTime)
{
HostLogger.LogDebug($"continue for outstanding requests.");
CombinedLogger.Flush();
continue;
}
else if (lastProcessedPosition > lastCheckpointedSequenceNumber)
{
await Task.WhenAll(Senders.Select(s => s.WaitForCurrentWorkToBeServiced()).ToList());
lastCheckpointedSequenceNumber = lastProcessedPosition;
await Save(storageConnectionString, process, leaseManager);
HostLogger.LogDebug($"continue for saving snapshot.");
continue;
}
else
{
// no more work to do here
HostLogger.LogInformation($"{(outOfTime ? "out of time" : "done")} after {stopwatch.Elapsed.TotalSeconds}s, {receiveCount + dedupCount + loopbackCount + clientCount} messages ({receiveCount} new, {loopbackCount} loopback, {clientCount} client, {dedupCount} deduped) in {iterationCount} batches");
CombinedLogger.Flush();
return(!outOfTime);
}
}
foreach (var ed in eventData)
{
var body = ed.Body;
var message = ProcessMessage.Deserialize(body.Array);
HostLogger.LogTrace($"{DateTime.UtcNow:o} Received {message}");
if (!message.IsExternalRequest && message.DeploymentTimestamp < deploymentTimestamp)
{
HostLogger.LogDebug($"Ignoring message from earlier deployment {message.DeploymentTimestamp}");
}
else if (!message.IsExternalRequest && message.DeploymentTimestamp > deploymentTimestamp)
{
HostLogger.LogError($"****** MISSING STATE ERROR process state is from older deployment, should have been initialized for new one! {message.DeploymentTimestamp} != {deploymentTimestamp}");
}
else if (message.IsSequenced &&
seenClock.HasSeen(message.Source, message.LastClock.Value))
{
dedupCount++;
HostLogger.LogTrace($"Deduping: {message}");
}
else if (message.IsExternalRequest)
{
clientCount++;
HostLogger.LogTrace($"Processing: {message}");
// deserialize content
List <IMessage> payload;
MemoryStream stream = new MemoryStream(message.Payload);
using (var binaryDictionaryReader = XmlDictionaryReader.CreateBinaryReader(stream, XmlDictionaryReaderQuotas.Max))
{
payload = (List <IMessage>)payloadSerializerLoopback.ReadObject(binaryDictionaryReader);
}
// Iterate
foreach (var m in payload)
{
process.ProcessMessage(m);
}
}
else if (message.IsLoopback)
{
loopbackCount++;
HostLogger.LogTrace($"Processing: {message}");
// deserialize content
List <IMessage> payload;
MemoryStream stream = new MemoryStream(message.Payload);
using (var binaryDictionaryReader = XmlDictionaryReader.CreateBinaryReader(stream, XmlDictionaryReaderQuotas.Max))
{
payload = (List <IMessage>)payloadSerializerLoopback.ReadObject(binaryDictionaryReader);
}
// Iterate
foreach (var m in payload)
{
process.ProcessMessage(m);
}
}
else
{
receiveCount++;
HostLogger.LogTrace($"Processing: {message}");
// deserialize content
List <KeyValuePair <long, IMessage> > payload;
MemoryStream stream = new MemoryStream(message.Payload);
using (var binaryDictionaryReader = XmlDictionaryReader.CreateBinaryReader(stream, XmlDictionaryReaderQuotas.Max))
{
payload = (List <KeyValuePair <long, IMessage> >)payloadSerializer.ReadObject(binaryDictionaryReader);
}
// Iterate
foreach (var kvp in payload)
{
if (!seenClock.HasSeen(message.Source, kvp.Key))
{
process.ProcessMessage(kvp.Value);
}
}
// Update seen clock.
seenClock.Set(message.Source, message.LastClock.Value);
}
lastProcessedPosition = ed.SystemProperties.SequenceNumber;
// Checkpoint and commit every X.
if (lastProcessedPosition > lastCheckpointedSequenceNumber &&
lastProcessedPosition % configuration.CheckpointInterval == configuration.CheckpointInterval - 1)
{
await Task.WhenAll(Senders.Select(s => s.WaitForCurrentWorkToBeServiced()).ToList());
// Checkpoint state.
lastCheckpointedSequenceNumber = lastProcessedPosition;
await Save(storageConnectionString, process, leaseManager);
}
}
}
catch (System.OperationCanceledException)
{
HostLogger.LogDebug($"receive returned OperationCanceledException, scheduling retry!");
continue;
}
}
}
finally
{
if (this.collectHostEvents)
{
this.blobTelemetryListener.OnApplicationEvent(processId, invocationId.ToString(), $"Process {processId}", "", OperationSide.Caller, OperationType.Host, (stopwatch.Elapsed - start).TotalMilliseconds);
}
}
}