/// <summary> /// This is expected to run for a long time, and it cannot leak exceptions /// </summary> private void Run() { try { var handles = new WaitHandle[] { _newEntry, _voterResponded, _promotableUpdated, _shutdownRequested }; var noopCmd = new DynamicJsonValue { ["Command"] = "noop" }; using (_engine.ContextPool.AllocateOperationContext(out TransactionOperationContext context)) using (var tx = context.OpenWriteTransaction()) using (var cmd = context.ReadObject(noopCmd, "noop-cmd")) { _engine.InsertToLeaderLog(context, Term, cmd, RachisEntryFlags.Noop); tx.Commit(); } _newEntry.Set(); //This is so the noop would register right away while (_running) { switch (WaitHandle.WaitAny(handles, _engine.ElectionTimeout)) { case 0: // new entry _newEntry.Reset(); // release any waiting ambassadors to send immediately TaskExecutor.CompleteAndReplace(ref _newEntriesArrived); if (_voters.Count == 0) { goto case 1; } break; case 1: // voter responded _voterResponded.Reset(); OnVoterConfirmation(); break; case 2: // promotable updated _promotableUpdated.Reset(); CheckPromotables(); break; case WaitHandle.WaitTimeout: break; case 3: // shutdown requested if (_engine.Log.IsInfoEnabled && _voters.Count != 0) { _engine.Log.Info($"{ToString()}: shutting down"); } _running.Lower(); return; } EnsureThatWeHaveLeadership(VotersMajority); _engine.ReportLeaderTime(LeaderShipDuration); // don't trancate if we are disposing an old peer // otherwise he would not recieve notification that he was // kick out of the cluster if (_previousPeersWereDisposed > 0) // Not Interlocked, because the race here is not interesting. { continue; } var lowestIndexInEntireCluster = GetLowestIndexInEntireCluster(); if (lowestIndexInEntireCluster != LowestIndexInEntireCluster) { using (_engine.ContextPool.AllocateOperationContext(out TransactionOperationContext context)) using (context.OpenWriteTransaction()) { _engine.TruncateLogBefore(context, lowestIndexInEntireCluster); LowestIndexInEntireCluster = lowestIndexInEntireCluster; context.Transaction.Commit(); } } } } catch (Exception e) { if (_engine.Log.IsInfoEnabled) { _engine.Log.Info("Error when running leader behavior", e); } if (e is VoronErrorException) { _engine.Notify(AlertRaised.Create( null, "Error when running leader behavior", e.Message, AlertType.ClusterTopologyWarning, NotificationSeverity.Error, details: new ExceptionDetails(e))); } try { _engine.SwitchToCandidateState("An error occurred during our leadership." + Environment.NewLine + e); } catch (Exception e2) { if (_engine.Log.IsOperationsEnabled) { _engine.Log.Operations("After leadership failure, could not setup switch to candidate state", e2); } } } }