public void Run(ICompiledApplication application)
{
TelemetryListenerArray telemetry = null;
if (application.Configurations.TryGetValue(typeof(ReactiveMachine.TelemetryBlobWriter.Configuration), out var c))
{
telemetry = new TelemetryListenerArray((ReactiveMachine.TelemetryBlobWriter.Configuration)c, application, this.GetType(), deploymentId, deploymentTimestamp);
application.HostServices.RegisterTelemetryListener(telemetry);
}
application.HostServices.RegisterSend(Send);
application.HostServices.RegisterGlobalExceptionHandler(HandleGlobalException);
_serializer = new DataContractSerializer(typeof(IMessage), application.SerializableTypes);
processes = new ProcessInfo[application.NumberProcesses];
for (uint i = 0; i < application.NumberProcesses; i++)
{
processes[i] = new ProcessInfo()
{
Inbox = new List <IMessage>(),
};
}
for (uint i = 0; i < application.NumberProcesses; i++)
{
processes[i].Process = application.MakeProcess(i);
processes[i].Process.FirstStart();
processes[i].Process.BecomePrimary();
}
Console.WriteLine($"=========================== START SIMULATION ===========================");
var stopwatch = new System.Diagnostics.Stopwatch();
stopwatch.Start();
while (processes[0].Process.RequestsOutstanding())
{
for (uint i = 0; i < application.NumberProcesses; i++)
{
var info = processes[i];
List <IMessage> deliver = empty;
lock (sgl)
{
if (info.Inbox.Count > 0)
{
deliver = info.Inbox;
info.Inbox = new List <IMessage>();
}
}
if (deliver.Count > 0)
{
foreach (var m in deliver)
{
//// lose 1/2 of activity responses that originated on older instances
//if (configuration.DeliverStaleExternalsOneOutOf != 1
// && m is ReactiveMachine.Implementation.RespondToActivity rte
// && rte.InstanceId != info.Process.InstanceId
// && (random.Next(configuration.DeliverStaleExternalsOneOutOf) != 0))
//{
// continue;
//}
info.Process.ProcessMessage(m);
if (configuration.RoundTripProcessStateEvery < int.MaxValue &&
random.Next(configuration.RoundTripProcessStateEvery) == 0) // for debugging process state serialization
{
info.Process.SaveState(out var bytes, out var label1);
info.Process.Restore(bytes, out var label2);
info.Process.BecomePrimary();
}
}
}
}
}
Console.WriteLine($"=========================== END SIMULATION ===========================");
stopwatch.Stop();
Console.WriteLine($"elapsed = {stopwatch.Elapsed.TotalSeconds:f2}s #messages={messageCount}");
telemetry?.Shutdown().Wait();
}
public async Task <bool> ResumeFromCheckpoint(LeaseManager leaseManager)
{
var start = stopwatch.Elapsed;
try
{
// kick off the load
var storageConnectionString = configuration.StorageConnectionString;
var loadtask = AzureBlobStorageStateManager.Load(storageConnectionString, HostLogger, processId);
// register host services
application.HostServices.RegisterSend(Send);
application.HostServices.RegisterApplicationLogger(ApplicationLogger);
application.HostServices.RegisterRuntimeLogger(RuntimeLogger);
// if (_configuration.AppInsightsInstrumentationKey != null)
// _application.HostServices.RegisterTelemetryListener(new ApplicationInsightsTelemetryListener(_configuration.AppInsightsInstrumentationKey, "eventhubs"));
if (blobTelemetryListener != null)
{
application.HostServices.RegisterTelemetryListener(blobTelemetryListener);
}
// read the checkpoint
var checkpointedState = await loadtask;
SetDeploymentTimestamp(checkpointedState.DeploymentTimestamp);
seenClock = checkpointedState.SeenClock;
ourClock = checkpointedState.OurClock;
var state = checkpointedState.State;
lastCheckpointedSequenceNumber = checkpointedState.Version;
HostLogger.LogDebug($"Resuming at position {lastCheckpointedSequenceNumber}");
Connections.ResumeFrom(lastCheckpointedSequenceNumber);
// build the process
IProcess process = application.MakeProcess(processId);
if (state == null)
{
process.FirstStart();
}
else
{
process.Restore(state, out var label);
}
process.BecomePrimary();
// start the message receiving loop
long lastProcessedPosition = 0;
int iterationCount = 0;
int dedupCount = 0;
int receiveCount = 0;
int loopbackCount = 0;
int clientCount = 0;
TimeSpan lastReport = stopwatch.Elapsed;
long lastReportedPosition = 0;
while (true)
{
iterationCount++;
if (lastProcessedPosition > lastReportedPosition && stopwatch.Elapsed - lastReport > TimeSpan.FromSeconds(15))
{
HostLogger.LogInformation($"progress to v{lastProcessedPosition} after {stopwatch.Elapsed.TotalSeconds:f2}s, {receiveCount + dedupCount + loopbackCount + clientCount} messages ({receiveCount} new, {loopbackCount} loopback, {clientCount} client, {dedupCount} deduped) in {iterationCount} batches");
lastReportedPosition = lastProcessedPosition;
lastReport = stopwatch.Elapsed;
CombinedLogger.Flush();
}
try
{
bool outOfTime = stopwatch.Elapsed > configuration.TimeLimit;
IEnumerable <EventData> eventData = outOfTime ? null :
await Connections.ProcessReceiver.ReceiveAsync(configuration.MaxReceiveBatchSize, iterationCount == 1?TimeSpan.FromSeconds(10) : configuration.ReceiveWaitTime);
if (eventData == null)
{
HostLogger.LogTrace($"{ DateTime.UtcNow:o} Received nothing. {Connections.ProcessReceiver.RuntimeInfo.LastSequenceNumber}");
if (process.RequestsOutstanding() && !outOfTime)
{
HostLogger.LogDebug($"continue for outstanding requests.");
CombinedLogger.Flush();
continue;
}
else if (lastProcessedPosition > lastCheckpointedSequenceNumber)
{
await Task.WhenAll(Senders.Select(s => s.WaitForCurrentWorkToBeServiced()).ToList());
lastCheckpointedSequenceNumber = lastProcessedPosition;
await Save(storageConnectionString, process, leaseManager);
HostLogger.LogDebug($"continue for saving snapshot.");
continue;
}
else
{
// no more work to do here
HostLogger.LogInformation($"{(outOfTime ? "out of time" : "done")} after {stopwatch.Elapsed.TotalSeconds}s, {receiveCount + dedupCount + loopbackCount + clientCount} messages ({receiveCount} new, {loopbackCount} loopback, {clientCount} client, {dedupCount} deduped) in {iterationCount} batches");
CombinedLogger.Flush();
return(!outOfTime);
}
}
foreach (var ed in eventData)
{
var body = ed.Body;
var message = ProcessMessage.Deserialize(body.Array);
HostLogger.LogTrace($"{DateTime.UtcNow:o} Received {message}");
if (!message.IsExternalRequest && message.DeploymentTimestamp < deploymentTimestamp)
{
HostLogger.LogDebug($"Ignoring message from earlier deployment {message.DeploymentTimestamp}");
}
else if (!message.IsExternalRequest && message.DeploymentTimestamp > deploymentTimestamp)
{
HostLogger.LogError($"****** MISSING STATE ERROR process state is from older deployment, should have been initialized for new one! {message.DeploymentTimestamp} != {deploymentTimestamp}");
}
else if (message.IsSequenced &&
seenClock.HasSeen(message.Source, message.LastClock.Value))
{
dedupCount++;
HostLogger.LogTrace($"Deduping: {message}");
}
else if (message.IsExternalRequest)
{
clientCount++;
HostLogger.LogTrace($"Processing: {message}");
// deserialize content
List <IMessage> payload;
MemoryStream stream = new MemoryStream(message.Payload);
using (var binaryDictionaryReader = XmlDictionaryReader.CreateBinaryReader(stream, XmlDictionaryReaderQuotas.Max))
{
payload = (List <IMessage>)payloadSerializerLoopback.ReadObject(binaryDictionaryReader);
}
// Iterate
foreach (var m in payload)
{
process.ProcessMessage(m);
}
}
else if (message.IsLoopback)
{
loopbackCount++;
HostLogger.LogTrace($"Processing: {message}");
// deserialize content
List <IMessage> payload;
MemoryStream stream = new MemoryStream(message.Payload);
using (var binaryDictionaryReader = XmlDictionaryReader.CreateBinaryReader(stream, XmlDictionaryReaderQuotas.Max))
{
payload = (List <IMessage>)payloadSerializerLoopback.ReadObject(binaryDictionaryReader);
}
// Iterate
foreach (var m in payload)
{
process.ProcessMessage(m);
}
}
else
{
receiveCount++;
HostLogger.LogTrace($"Processing: {message}");
// deserialize content
List <KeyValuePair <long, IMessage> > payload;
MemoryStream stream = new MemoryStream(message.Payload);
using (var binaryDictionaryReader = XmlDictionaryReader.CreateBinaryReader(stream, XmlDictionaryReaderQuotas.Max))
{
payload = (List <KeyValuePair <long, IMessage> >)payloadSerializer.ReadObject(binaryDictionaryReader);
}
// Iterate
foreach (var kvp in payload)
{
if (!seenClock.HasSeen(message.Source, kvp.Key))
{
process.ProcessMessage(kvp.Value);
}
}
// Update seen clock.
seenClock.Set(message.Source, message.LastClock.Value);
}
lastProcessedPosition = ed.SystemProperties.SequenceNumber;
// Checkpoint and commit every X.
if (lastProcessedPosition > lastCheckpointedSequenceNumber &&
lastProcessedPosition % configuration.CheckpointInterval == configuration.CheckpointInterval - 1)
{
await Task.WhenAll(Senders.Select(s => s.WaitForCurrentWorkToBeServiced()).ToList());
// Checkpoint state.
lastCheckpointedSequenceNumber = lastProcessedPosition;
await Save(storageConnectionString, process, leaseManager);
}
}
}
catch (System.OperationCanceledException)
{
HostLogger.LogDebug($"receive returned OperationCanceledException, scheduling retry!");
continue;
}
}
}
finally
{
if (this.collectHostEvents)
{
this.blobTelemetryListener.OnApplicationEvent(processId, invocationId.ToString(), $"Process {processId}", "", OperationSide.Caller, OperationType.Host, (stopwatch.Elapsed - start).TotalMilliseconds);
}
}
}
public void Run(ICompiledApplication application)
{
TelemetryListenerArray telemetry = null;
if (application.Configurations.TryGetValue(typeof(ReactiveMachine.TelemetryBlobWriter.Configuration), out var c))
{
telemetry = new TelemetryListenerArray((ReactiveMachine.TelemetryBlobWriter.Configuration)c, application, this.GetType(), deploymentId, deploymentTimestamp);
application.HostServices.RegisterTelemetryListener(telemetry);
}
application.HostServices.RegisterSend(Send);
application.HostServices.RegisterGlobalExceptionHandler(HandleGlobalException);
application.HostServices.RegisterGlobalShutdown(() => shutdown.Cancel());
_serializer = new DataContractSerializer(typeof(IMessage), application.SerializableTypes);
processes = new ProcessInfo[application.NumberProcesses];
for (uint i = 0; i < application.NumberProcesses; i++)
{
processes[i] = new ProcessInfo()
{
Inbox = new List <IMessage>(),
};
}
for (uint i = 0; i < application.NumberProcesses; i++)
{
var info = processes[i];
logger.LogDebug($"Recovering process {i}.");
info.Process = application.MakeProcess(i);
info.Process.FirstStart();
}
for (uint i = 0; i < application.NumberProcesses; i++)
{
uint processId = i;
new Thread(() => RunProcess(processId, processes[processId], application)).Start();
}
bool someoneBusy = true;
while (!shutdown.IsCancellationRequested)
{
someoneBusy = false;
for (uint i = 0; i < application.NumberProcesses; i++)
{
var info = processes[i];
long received;
bool busy;
lock (info)
{
received = info.Received;
busy = (info.Inbox.Count > 0) || (info.Process?.RequestsOutstanding() ?? true);
}
someoneBusy = busy || someoneBusy;
logger.LogInformation($"Process {i}: Received={received:D12} busy={busy}");
}
if (!someoneBusy)
{
break;
}
emulator.FlushLog();
shutdown.Token.WaitHandle.WaitOne(TimeSpan.FromSeconds(10));
}
telemetry?.Shutdown().Wait();
}