public void Run(ICompiledApplication application) { TelemetryListenerArray telemetry = null; if (application.Configurations.TryGetValue(typeof(ReactiveMachine.TelemetryBlobWriter.Configuration), out var c)) { telemetry = new TelemetryListenerArray((ReactiveMachine.TelemetryBlobWriter.Configuration)c, application, this.GetType(), deploymentId, deploymentTimestamp); application.HostServices.RegisterTelemetryListener(telemetry); } application.HostServices.RegisterSend(Send); application.HostServices.RegisterGlobalExceptionHandler(HandleGlobalException); _serializer = new DataContractSerializer(typeof(IMessage), application.SerializableTypes); processes = new ProcessInfo[application.NumberProcesses]; for (uint i = 0; i < application.NumberProcesses; i++) { processes[i] = new ProcessInfo() { Inbox = new List <IMessage>(), }; } for (uint i = 0; i < application.NumberProcesses; i++) { processes[i].Process = application.MakeProcess(i); processes[i].Process.FirstStart(); processes[i].Process.BecomePrimary(); } Console.WriteLine($"=========================== START SIMULATION ==========================="); var stopwatch = new System.Diagnostics.Stopwatch(); stopwatch.Start(); while (processes[0].Process.RequestsOutstanding()) { for (uint i = 0; i < application.NumberProcesses; i++) { var info = processes[i]; List <IMessage> deliver = empty; lock (sgl) { if (info.Inbox.Count > 0) { deliver = info.Inbox; info.Inbox = new List <IMessage>(); } } if (deliver.Count > 0) { foreach (var m in deliver) { //// lose 1/2 of activity responses that originated on older instances //if (configuration.DeliverStaleExternalsOneOutOf != 1 // && m is ReactiveMachine.Implementation.RespondToActivity rte // && rte.InstanceId != info.Process.InstanceId // && (random.Next(configuration.DeliverStaleExternalsOneOutOf) != 0)) //{ // continue; //} info.Process.ProcessMessage(m); if (configuration.RoundTripProcessStateEvery < int.MaxValue && random.Next(configuration.RoundTripProcessStateEvery) == 0) // for debugging process state serialization { info.Process.SaveState(out var bytes, out var label1); info.Process.Restore(bytes, out var label2); info.Process.BecomePrimary(); } } } } } Console.WriteLine($"=========================== END SIMULATION ==========================="); stopwatch.Stop(); Console.WriteLine($"elapsed = {stopwatch.Elapsed.TotalSeconds:f2}s #messages={messageCount}"); telemetry?.Shutdown().Wait(); }
public async Task <bool> ResumeFromCheckpoint(LeaseManager leaseManager) { var start = stopwatch.Elapsed; try { // kick off the load var storageConnectionString = configuration.StorageConnectionString; var loadtask = AzureBlobStorageStateManager.Load(storageConnectionString, HostLogger, processId); // register host services application.HostServices.RegisterSend(Send); application.HostServices.RegisterApplicationLogger(ApplicationLogger); application.HostServices.RegisterRuntimeLogger(RuntimeLogger); // if (_configuration.AppInsightsInstrumentationKey != null) // _application.HostServices.RegisterTelemetryListener(new ApplicationInsightsTelemetryListener(_configuration.AppInsightsInstrumentationKey, "eventhubs")); if (blobTelemetryListener != null) { application.HostServices.RegisterTelemetryListener(blobTelemetryListener); } // read the checkpoint var checkpointedState = await loadtask; SetDeploymentTimestamp(checkpointedState.DeploymentTimestamp); seenClock = checkpointedState.SeenClock; ourClock = checkpointedState.OurClock; var state = checkpointedState.State; lastCheckpointedSequenceNumber = checkpointedState.Version; HostLogger.LogDebug($"Resuming at position {lastCheckpointedSequenceNumber}"); Connections.ResumeFrom(lastCheckpointedSequenceNumber); // build the process IProcess process = application.MakeProcess(processId); if (state == null) { process.FirstStart(); } else { process.Restore(state, out var label); } process.BecomePrimary(); // start the message receiving loop long lastProcessedPosition = 0; int iterationCount = 0; int dedupCount = 0; int receiveCount = 0; int loopbackCount = 0; int clientCount = 0; TimeSpan lastReport = stopwatch.Elapsed; long lastReportedPosition = 0; while (true) { iterationCount++; if (lastProcessedPosition > lastReportedPosition && stopwatch.Elapsed - lastReport > TimeSpan.FromSeconds(15)) { HostLogger.LogInformation($"progress to v{lastProcessedPosition} after {stopwatch.Elapsed.TotalSeconds:f2}s, {receiveCount + dedupCount + loopbackCount + clientCount} messages ({receiveCount} new, {loopbackCount} loopback, {clientCount} client, {dedupCount} deduped) in {iterationCount} batches"); lastReportedPosition = lastProcessedPosition; lastReport = stopwatch.Elapsed; CombinedLogger.Flush(); } try { bool outOfTime = stopwatch.Elapsed > configuration.TimeLimit; IEnumerable <EventData> eventData = outOfTime ? null : await Connections.ProcessReceiver.ReceiveAsync(configuration.MaxReceiveBatchSize, iterationCount == 1?TimeSpan.FromSeconds(10) : configuration.ReceiveWaitTime); if (eventData == null) { HostLogger.LogTrace($"{ DateTime.UtcNow:o} Received nothing. {Connections.ProcessReceiver.RuntimeInfo.LastSequenceNumber}"); if (process.RequestsOutstanding() && !outOfTime) { HostLogger.LogDebug($"continue for outstanding requests."); CombinedLogger.Flush(); continue; } else if (lastProcessedPosition > lastCheckpointedSequenceNumber) { await Task.WhenAll(Senders.Select(s => s.WaitForCurrentWorkToBeServiced()).ToList()); lastCheckpointedSequenceNumber = lastProcessedPosition; await Save(storageConnectionString, process, leaseManager); HostLogger.LogDebug($"continue for saving snapshot."); continue; } else { // no more work to do here HostLogger.LogInformation($"{(outOfTime ? "out of time" : "done")} after {stopwatch.Elapsed.TotalSeconds}s, {receiveCount + dedupCount + loopbackCount + clientCount} messages ({receiveCount} new, {loopbackCount} loopback, {clientCount} client, {dedupCount} deduped) in {iterationCount} batches"); CombinedLogger.Flush(); return(!outOfTime); } } foreach (var ed in eventData) { var body = ed.Body; var message = ProcessMessage.Deserialize(body.Array); HostLogger.LogTrace($"{DateTime.UtcNow:o} Received {message}"); if (!message.IsExternalRequest && message.DeploymentTimestamp < deploymentTimestamp) { HostLogger.LogDebug($"Ignoring message from earlier deployment {message.DeploymentTimestamp}"); } else if (!message.IsExternalRequest && message.DeploymentTimestamp > deploymentTimestamp) { HostLogger.LogError($"****** MISSING STATE ERROR process state is from older deployment, should have been initialized for new one! {message.DeploymentTimestamp} != {deploymentTimestamp}"); } else if (message.IsSequenced && seenClock.HasSeen(message.Source, message.LastClock.Value)) { dedupCount++; HostLogger.LogTrace($"Deduping: {message}"); } else if (message.IsExternalRequest) { clientCount++; HostLogger.LogTrace($"Processing: {message}"); // deserialize content List <IMessage> payload; MemoryStream stream = new MemoryStream(message.Payload); using (var binaryDictionaryReader = XmlDictionaryReader.CreateBinaryReader(stream, XmlDictionaryReaderQuotas.Max)) { payload = (List <IMessage>)payloadSerializerLoopback.ReadObject(binaryDictionaryReader); } // Iterate foreach (var m in payload) { process.ProcessMessage(m); } } else if (message.IsLoopback) { loopbackCount++; HostLogger.LogTrace($"Processing: {message}"); // deserialize content List <IMessage> payload; MemoryStream stream = new MemoryStream(message.Payload); using (var binaryDictionaryReader = XmlDictionaryReader.CreateBinaryReader(stream, XmlDictionaryReaderQuotas.Max)) { payload = (List <IMessage>)payloadSerializerLoopback.ReadObject(binaryDictionaryReader); } // Iterate foreach (var m in payload) { process.ProcessMessage(m); } } else { receiveCount++; HostLogger.LogTrace($"Processing: {message}"); // deserialize content List <KeyValuePair <long, IMessage> > payload; MemoryStream stream = new MemoryStream(message.Payload); using (var binaryDictionaryReader = XmlDictionaryReader.CreateBinaryReader(stream, XmlDictionaryReaderQuotas.Max)) { payload = (List <KeyValuePair <long, IMessage> >)payloadSerializer.ReadObject(binaryDictionaryReader); } // Iterate foreach (var kvp in payload) { if (!seenClock.HasSeen(message.Source, kvp.Key)) { process.ProcessMessage(kvp.Value); } } // Update seen clock. seenClock.Set(message.Source, message.LastClock.Value); } lastProcessedPosition = ed.SystemProperties.SequenceNumber; // Checkpoint and commit every X. if (lastProcessedPosition > lastCheckpointedSequenceNumber && lastProcessedPosition % configuration.CheckpointInterval == configuration.CheckpointInterval - 1) { await Task.WhenAll(Senders.Select(s => s.WaitForCurrentWorkToBeServiced()).ToList()); // Checkpoint state. lastCheckpointedSequenceNumber = lastProcessedPosition; await Save(storageConnectionString, process, leaseManager); } } } catch (System.OperationCanceledException) { HostLogger.LogDebug($"receive returned OperationCanceledException, scheduling retry!"); continue; } } } finally { if (this.collectHostEvents) { this.blobTelemetryListener.OnApplicationEvent(processId, invocationId.ToString(), $"Process {processId}", "", OperationSide.Caller, OperationType.Host, (stopwatch.Elapsed - start).TotalMilliseconds); } } }
public void Run(ICompiledApplication application) { TelemetryListenerArray telemetry = null; if (application.Configurations.TryGetValue(typeof(ReactiveMachine.TelemetryBlobWriter.Configuration), out var c)) { telemetry = new TelemetryListenerArray((ReactiveMachine.TelemetryBlobWriter.Configuration)c, application, this.GetType(), deploymentId, deploymentTimestamp); application.HostServices.RegisterTelemetryListener(telemetry); } application.HostServices.RegisterSend(Send); application.HostServices.RegisterGlobalExceptionHandler(HandleGlobalException); application.HostServices.RegisterGlobalShutdown(() => shutdown.Cancel()); _serializer = new DataContractSerializer(typeof(IMessage), application.SerializableTypes); processes = new ProcessInfo[application.NumberProcesses]; for (uint i = 0; i < application.NumberProcesses; i++) { processes[i] = new ProcessInfo() { Inbox = new List <IMessage>(), }; } for (uint i = 0; i < application.NumberProcesses; i++) { var info = processes[i]; logger.LogDebug($"Recovering process {i}."); info.Process = application.MakeProcess(i); info.Process.FirstStart(); } for (uint i = 0; i < application.NumberProcesses; i++) { uint processId = i; new Thread(() => RunProcess(processId, processes[processId], application)).Start(); } bool someoneBusy = true; while (!shutdown.IsCancellationRequested) { someoneBusy = false; for (uint i = 0; i < application.NumberProcesses; i++) { var info = processes[i]; long received; bool busy; lock (info) { received = info.Received; busy = (info.Inbox.Count > 0) || (info.Process?.RequestsOutstanding() ?? true); } someoneBusy = busy || someoneBusy; logger.LogInformation($"Process {i}: Received={received:D12} busy={busy}"); } if (!someoneBusy) { break; } emulator.FlushLog(); shutdown.Token.WaitHandle.WaitOne(TimeSpan.FromSeconds(10)); } telemetry?.Shutdown().Wait(); }