public NormalStateMessageProcessor(ReplicaState replicaState, MessageServiceClient messageServiceClient)
{
this.messageServiceClient = messageServiceClient;
this.replicaState = replicaState;
Log.Info("Changed to Normal State.");
}
public ViewChangeMessageProcessor(
MessageServiceClient messageServiceClient,
ReplicaState replicaState,
DoViewChangeXL doViewChange)
{
this.messageServiceClient = messageServiceClient;
this.replicaState = replicaState;
this.viewNumber = doViewChange.ViewNumber;
this.configuration = doViewChange.Configuration;
this.imTheManager = doViewChange.Configuration.Values.ToArray()[0].Equals(this.replicaState.MyUrl);
this.numberToWait = (doViewChange.Configuration.Count - 1) / 2;
this.messagesDoViewChange = 0;
this.bestDoViewChange = new DoViewChangeXL(
this.replicaState.ServerId,
this.viewNumber,
this.replicaState.ViewNumber,
this.configuration,
this.replicaState.TupleSpace,
this.replicaState.ClientTable,
this.replicaState.CommitNumber);
Log.Info("Changed to View Change State.");
// Stay in this state for a timeout
Task.Factory.StartNew(this.StartTimeout);
}
public ViewChangeMessageProcessor(
MessageServiceClient messageServiceClient,
ReplicaState replicaState,
int viewNumber,
SortedDictionary <string, Uri> configuration)
{
this.messageServiceClient = messageServiceClient;
this.replicaState = replicaState;
this.viewNumber = viewNumber;
this.configuration = configuration;
this.imTheManager = this.configuration.Values.ToArray()[0].Equals(this.replicaState.MyUrl);
this.numberToWait = this.replicaState.Configuration.Count / 2;
this.messagesDoViewChange = 0;
this.bestDoViewChange = new DoViewChangeXL(
this.replicaState.ServerId,
this.viewNumber,
this.replicaState.ViewNumber,
this.configuration,
this.replicaState.TupleSpace,
this.replicaState.ClientTable,
this.replicaState.CommitNumber);
Log.Info("Changed to View Change State.");
// Start the view change protocol
Task.Factory.StartNew(this.MulticastStartViewChange);
// Stay in this state for a timeout
Task.Factory.StartNew(this.StartTimeout);
}
private ClientRequest cleanProposal(ProposalDecision decision, ReplicaState state)
{
ClientRequest request = state.ProposalsRequestsBySlotId[decision.SlotNumber];
state.ProposalsRequestsBySlotId.Remove(decision.SlotNumber);
return(request);
}
private void ChangeState(ReplicaState state)
{
_replica.Service.ReplicaEvents.OnNext(new ReplicaEvent(state, _replica));
lock (_stateChangeLocker)
{
_currentState = state;
_lastStateChange = DateTime.Now;
}
}
public RecoveryStateMessageProcessor(ReplicaState replicaState, MessageServiceClient messageServiceClient)
{
this.messageServiceClient = messageServiceClient;
this.replicaState = replicaState;
Log.Info("Changed to Recovery State.");
// Start the protocol
Task.Factory.StartNew(this.RecoveryProtocol);
}
public ReplicaMonitorState(ReplicaStatus replica, ILogger logger)
{
_replica = replica;
_logger = logger;
_currentState = ReplicaState.Started;
_lastStateChange = DateTime.Now;
_stateChangeLocker = new object();
Init();
}
public InitializationStateMessageProcessor(ReplicaState replicaState, MessageServiceClient messageServiceClient)
{
this.messageServiceClient = messageServiceClient;
this.replicaState = replicaState;
Log.Info("Changed to Initialization State.");
Task.Factory.StartNew(this.InitProtocol);
Task.Factory.StartNew(this.StartTimeout);
}
private void StartLivenessProbe(Probe probe, ReplicaState moveToOnSuccess = ReplicaState.Healthy)
{
// currently only HTTP is available
if (probe.Http == null)
{
_logger.LogWarning("Cannot start probing replica {name} because probe configuration is not set", _replica.Name);
return;
}
_livenessProber = new HttpProber(_replica, "liveness", probe, probe.Http, _logger);
var failureThreshold = probe.FailureThreshold;
var failures = 0;
var dead = false;
_livenessProberObserver = _livenessProber.ProbeResults.Subscribe(entry =>
{
if (dead)
{
return;
}
if (entry)
{
// Reset failures count on success
failures = 0;
}
(var currentState, _) = ReadCurrentState();
var failuresPastThreshold = failures >= failureThreshold;
switch ((entry, currentState, moveToOnSuccess, failuresPastThreshold))
{
case (false, _, _, true):
dead = true;
Kill();
break;
case (false, _, _, false):
Interlocked.Increment(ref failures);
break;
case (true, ReplicaState.Started, ReplicaState.Ready, _):
case (true, ReplicaState.Healthy, ReplicaState.Ready, _):
MoveToReady();
break;
case (true, ReplicaState.Started, ReplicaState.Healthy, _):
MoveToHealthy(from: ReplicaState.Started);
break;
}
});
_livenessProber.Start();
}
public void Execute(MessageStrategyExecuteArg <IMessage> obj)
{
if (!(obj.Message is ClientRequest))
{
throw new MessageStrategyException("This strategy shouldn't be invoked with this message type");
}
ClientRequest request = obj.Message as ClientRequest;
ReplicaState state = obj.RoleState as ReplicaState;
sendProposalsToLeaders(request, state);
}
private void sendProposalsToLeaders(ClientRequest request, ReplicaState state)
{
state.ProposalsRequestsBySlotId.Add(state.FirstUnusedSlot, request);
state.ClientsPendingResponseBySlotId.Add(state.FirstUnusedSlot, request);
ProposalRequest proposal = new ProposalRequest();
proposal.SlotNumber = state.FirstUnusedSlot;
proposal.Command = request.Command;
proposal.MessageSender = state.MessageSender;
foreach (MessageSender leader in state.Leaders)
{
this.broker.SendMessage(leader.UniqueId, proposal);
}
state.FirstUnusedSlot++;
}
/// <summary>
/// Called on the replica when it is being aborted.
/// </summary>
public virtual void Abort()
{
AppTrace.TraceSource.WriteNoise("StatefulServiceReplica.Abort", "{0}", this.ToString());
//
// Perform local abort functionality. Abort state provider broker.
//
this.stateProviderBroker.Abort();
//
// Perform custom abort functionality.
//
this.OnAbort();
//
// Change current replica state.
//
this.replicaState = ReplicaState.Aborted;
AppTrace.TraceSource.WriteNoise("StatefulServiceReplica.State", "{0} is {1}", this.ToString(), this.replicaState);
}
/// <summary>
/// Called on the replica when it is being closed.
/// </summary>
/// <param name="cancellationToken">Propagates notification that operation should be canceled.</param>
/// <returns></returns>
public virtual async Task CloseAsync(CancellationToken cancellationToken)
{
AppTrace.TraceSource.WriteNoise("StatefulServiceReplica.Close", "{0}", this.ToString());
//
// Perform local close functionality.Close state provider broker.
//
await this.stateProviderBroker.CloseAsync(cancellationToken);
//
// Perform custom close functionality.
//
await this.OnCloseAsync(cancellationToken);
//
// Change current replica state.
//
this.replicaState = ReplicaState.Closed;
AppTrace.TraceSource.WriteNoise("StatefulServiceReplica.State", "{0} is {1}", this.ToString(), this.replicaState);
}
/// <summary>
/// Abort is called when the replica is being forcibly closed
/// </summary>
void IStatefulServiceReplica.Abort()
{
try
{
FabricEvents.Events.Lifecycle(
this.tracer.Type,
"Abort: Aborting replica " + this.initializationParameters.ReplicaId);
this.transactionalReplicator.Abort();
this.OnAbort();
this.replicaState = ReplicaState.Aborted;
FabricEvents.Events.Lifecycle(
this.tracer.Type,
"Abort: Finished aborting replica " + this.initializationParameters.ReplicaId);
}
finally
{
TaskScheduler.UnobservedTaskException -= this.ProcessUnobservedTaskException;
}
}
public void Execute(MessageStrategyExecuteArg <IMessage> obj)
{
if (!(obj.Message is ProposalDecision))
{
throw new MessageStrategyException("This strategy shouldn't be invoked with this message type");
}
ProposalDecision decision = obj.Message as ProposalDecision;
ReplicaState state = obj.RoleState as ReplicaState;
storeDecision(decision, state);
if (proposalHasBeenApproved(decision, state))
{
ClientRequest clientRequest = cleanProposal(decision, state);
OnDecisionApproved?.Invoke(this, clientRequest);
}
else
{
temporaryRemoveClientPendingResponseUntilNextRetry(decision, state);
ClientRequest clientRequest = cleanProposal(decision, state);
OnDecisionRejected?.Invoke(this, clientRequest);
}
}
public void LoadState(ReplicaState state)
{
if (state.OperationInternalState != null)
{
ProcessFunction.InternalState = state.OperationInternalState;
}
foreach (var d in state.OutputStreamsIds)
{
destinations[d.Key].LoadState(d.Value);
}
foreach (var opName in state.InputStreamsIds.Keys)
{
var sentids = state.InputStreamsIds[opName].SentIds;
for (int i = 0; i < sentids.Count; i++)
{
originOperators[opName][i].LastProcessedId = sentids[i];
}
}
StartFrom = state.LastEmittedTuple;
LastProcessedId = state.LastProcessedId;
freezingState = state.IsFrozen;
processingState = state.IsStarted;
}
/// <summary>
/// CloseAsync is called when the replica is going to be closed
/// </summary>
/// <param name="cancellationToken">Cancellation token.</param>
/// <returns>Task representing the asynchronous operation.</returns>
async Task IStatefulServiceReplica.CloseAsync(CancellationToken cancellationToken)
{
try
{
FabricEvents.Events.Lifecycle(
this.tracer.Type,
"CloseAsync: Closing replica " + this.initializationParameters.ReplicaId);
await this.transactionalReplicator.CloseAsync().ConfigureAwait(false);
await this.OnCloseAsync(cancellationToken).ConfigureAwait(false);
this.replicaState = ReplicaState.Closed;
FabricEvents.Events.Lifecycle(
this.tracer.Type,
"CloseAsync: Finished closing replica " + this.initializationParameters.ReplicaId);
}
finally
{
TaskScheduler.UnobservedTaskException -= this.ProcessUnobservedTaskException;
}
}
public ReplicaState GetState()
{
//Console.WriteLine("Taking snapshot");
lock (this)
{
var result = new ReplicaState();
if (result.OperationInternalState != null && result.OperationInternalState.GetType().IsSerializable)
{
result.OperationInternalState = ProcessFunction.InternalState;
}
result.OutputStreamsIds = new Dictionary <string, DestinationState>();
result.InputStreamsIds = new Dictionary <string, OriginState>();
foreach (var inputstream in originOperators)
{
result.InputStreamsIds[inputstream.Key] = new OriginState
{
SentIds = inputstream.Value.Select((x) => x.LastProcessedId).ToList()
};
}
foreach (var d in destinations)
{
var state = d.Value.GetState();
if (state != null)
{
result.OutputStreamsIds[d.Key] = state;
}
}
result.LastEmittedTuple = TupleCounter;
result.LastProcessedId = LastProcessedId;
result.IsStarted = processingState;
result.IsFrozen = freezingState;
return(result);
}
}
public ReplicaEvent(ReplicaState state, ReplicaStatus replica)
{
State = state;
Replica = replica;
}
public static async Task StartHostAndWaitForReplicasToStart(TyeHost host, string[] services = null, ReplicaState desiredState = ReplicaState.Started)
{
if (services == null)
{
await DoOperationAndWaitForReplicasToChangeState(host, desiredState, host.Application.Services.Sum(s => s.Value.Description.Replicas), null, null, TimeSpan.Zero, h => h.StartAsync());
}
else
{
if (services.Any(s => !host.Application.Services.ContainsKey(s)))
{
throw new ArgumentException($"not all services given in {nameof(services)} exist");
}
await DoOperationAndWaitForReplicasToChangeState(host, desiredState, host.Application.Services.Where(s => services.Contains(s.Value.Description.Name)).Sum(s => s.Value.Description.Replicas), services.ToHashSet(), null, ev => ev.Replica.Service.Description.Name, TimeSpan.Zero, h => h.StartAsync());
}
}
public static Task <bool> DoOperationAndWaitForReplicasToChangeState(TyeHost host, ReplicaState desiredState, int n, HashSet <string> toChange, HashSet <string> rest, TimeSpan waitUntilSuccess, Func <TyeHost, Task> operation) => DoOperationAndWaitForReplicasToChangeState(host, desiredState, n, toChange, rest, ev => ev.Replica.Name, waitUntilSuccess, operation);
public static async Task <bool> DoOperationAndWaitForReplicasToChangeState(TyeHost host, ReplicaState desiredState, int n, HashSet <string> toChange, HashSet <string> rest, Func <ReplicaEvent, string> entitySelector, TimeSpan waitUntilSuccess, Func <TyeHost, Task> operation)
{
if (toChange != null && rest != null && rest.Overlaps(toChange))
{
throw new ArgumentException($"{nameof(toChange)} and {nameof(rest)} can't overlap");
}
var changedTask = new TaskCompletionSource <bool>();
var remaining = n;
void OnReplicaChange(ReplicaEvent ev)
{
if (rest != null && rest.Contains(entitySelector(ev)))
{
changedTask !.TrySetResult(false);
}
else if ((toChange == null || toChange.Contains(entitySelector(ev))) && ev.State == desiredState)
{
Interlocked.Decrement(ref remaining);
}
if (remaining == 0)
{
Task.Delay(waitUntilSuccess)
.ContinueWith(_ =>
{
if (!changedTask !.Task.IsCompleted)
{
changedTask !.TrySetResult(remaining == 0);
}
});
}
}
var servicesStateObserver = host.Application.Services.Select(srv => srv.Value.ReplicaEvents.Subscribe(OnReplicaChange)).ToList();
await operation(host);
using var cancellation = new CancellationTokenSource(WaitForServicesTimeout);
try
{
await using (cancellation.Token.Register(() => changedTask.TrySetCanceled()))
{
return(await changedTask.Task);
}
}
finally
{
foreach (var observer in servicesStateObserver)
{
observer.Dispose();
}
}
}
private bool proposalHasBeenApproved(ProposalDecision decision, ReplicaState state)
{
return(state.ProposalsRequestsBySlotId.ContainsKey(decision.SlotNumber) &&
state.DecisionsBySlotId.ContainsKey(decision.SlotNumber) &&
state.ProposalsRequestsBySlotId[decision.SlotNumber].Command == state.DecisionsBySlotId[decision.SlotNumber]);
}
private void temporaryRemoveClientPendingResponseUntilNextRetry(ProposalDecision decision, ReplicaState state)
{
state.ClientsPendingResponseBySlotId.Remove(decision.SlotNumber);
}
private bool replicaIsProposalEmitter(ProposalDecision decision, ReplicaState state)
{
return(state.ProposalsRequestsBySlotId.ContainsKey(decision.SlotNumber));
}
public void Recover(int failedId)
{
Dictionary <int, Replica> replicasCopy;
lock (replicas)
{
replicasCopy = new Dictionary <int, Replica>(replicas);
}
Replica r = CreateReplica(failedId);
ReplicaState repState = otherReplicasStates[failedId]; //get the last state of crashed replica
Dictionary <string, OriginState> os = repState.InputStreamsIds;
Console.WriteLine($"Started to recover replica {failedId} from state {repState}");
r.LoadState(repState);
AddReplica(r);
allReplicas[failedId] = this;
foreach (string opName in os.Keys)
{
var sentIds = os[opName].SentIds; // Only keeps the last id sent to each destination
//for each operator ask a re-sent
if (opName == this.info.ID)
{
continue;
}
for (int j = 0; j < sentIds.Count; j++)
{
while (true)
{
try
{
Console.WriteLine($"Asking for resend to {opName} ({j})");
inputReplicas[opName][j].Resend(sentIds[j], this.info.ID, failedId, j, SelfURL);
break;
}
catch (Exception e)
{
Console.WriteLine("Resending failed. Trying again. Stay positive." + e.Message);
}
}
}
}
// Console.WriteLine("Phase 1 completed: Tuples were resent.");
foreach (string opName in inputReplicas.Keys)
{
for (int i = 0; i < inputReplicas[opName].Count; i++)
{
try
{
inputReplicas[opName][i].ReRoute(this.adresses[failedId], this.SelfURL);
}
catch (Exception e)
{
Console.WriteLine("ReplicaManager.Recover: Reroute of input replicas failed " + e.Message);
}
}
}
// Console.WriteLine("Phase 2 completed: Input replicas were rerouted.");
foreach (string opName in outputReplicas.Keys)
{
for (int i = 0; i < outputReplicas[opName].Count; i++)
{
try
{
outputReplicas[opName][i].ReRoute(this.adresses[failedId], this.SelfURL);
}
catch (Exception e)
{
Console.WriteLine("ReplicaManager.Recover: Reroute of output replicas failed" + e.Message);
}
}
}
// Console.WriteLine("Phase 3 completed: Output replicas were rerouted.");
try
{
puppetMaster.ReRoute(this.info.ID, failedId, this.SelfURL);
}
catch (Exception e)
{
Console.WriteLine("ReplicaManager.Recover: Reroute of puppet master failed" + e.Message);
}
//Console.WriteLine("Phase 4 completed: Puppet master was rerouted.");
adresses[failedId] = SelfURL;
Console.WriteLine("MISSION COMPLETED: all recovered!");
if (repState.IsFrozen)
{
r.Freeze();
}
if (replicas.First().Value.processingState)
{
r.Start();
}
r.Init();
//resend
}
public void SendState(ReplicaState state, int id)
{
otherReplicasStates[id] = state;
}
private void storeDecision(ProposalDecision decision, ReplicaState state)
{
state.DecisionsBySlotId.Add(decision.SlotNumber, decision.Command);
}
private void MoveToHealthy(ReplicaState from)
{
_logger.LogInformation("Replica {name} is moving to an healthy state", _replica.Name);
ChangeState(ReplicaState.Healthy);
}
/// <summary>
/// Opens the replica.
/// </summary>
/// <param name="openMode">Replica open mode (new or existent).</param>
/// <param name="partition">Stateful partition object.</param>
/// <param name="cancellationToken">Propagates notification that operations should be canceled.</param>
/// <returns></returns>
public virtual async Task <IReplicator> OpenAsync(ReplicaOpenMode openMode, IStatefulServicePartition partition, CancellationToken cancellationToken)
{
//
// Check arguments.
//
if (null == partition)
{
AppTrace.TraceSource.WriteError("StatefulServiceReplica.Open", "{0}", this.ToString());
throw new ArgumentNullException("partition");
}
//
// Perform local open functionality.
//
AppTrace.TraceSource.WriteNoise("StatefulServiceReplica.Open", "{0}", this.ToString());
//
// Set partition related members.
//
this.servicePartition = partition;
this.serviceGroupPartition = partition as IServiceGroupPartition;
this.servicePartitionEx = partition as IStatefulServicePartitionEx;
//
// Create an implementation of the state provider broker.
//
this.stateProviderBroker = this.CreateStateProviderBroker();
if (null == this.stateProviderBroker)
{
AppTrace.TraceSource.WriteError("StatefulServiceReplica.Open", "{0} invalid state provider broker", this.ToString());
throw new InvalidOperationException();
}
//
// Extract state providers.
//
this.stateProvider = this.stateProviderBroker as IStateProvider;
this.atomicGroupStateProvider = this.stateProviderBroker as IAtomicGroupStateProvider;
this.atomicGroupStateProviderEx = this.stateProviderBroker as IAtomicGroupStateProviderEx;
if (null == this.stateProvider && null == this.atomicGroupStateProvider && null == this.atomicGroupStateProviderEx)
{
AppTrace.TraceSource.WriteError("StatefulServiceReplica.Open", "{0} invalid state providers", this.ToString());
throw new InvalidOperationException();
}
//
// Create replicator settings (replication and log).
// For service groups, these settings are specified in the service manifest.
//
ReplicatorSettings replicatorSettings = null;
if (null != this.serviceGroupPartition)
{
replicatorSettings = this.ReplicatorSettings;
}
ReplicatorLogSettings replicatorLogSettings = null;
if (null != this.atomicGroupStateProviderEx && null != this.serviceGroupPartition)
{
replicatorLogSettings = this.ReplicatorLogSettings;
}
//
// Create replicator.
//
FabricReplicator replicator = null;
FabricReplicatorEx replicatorEx = null;
if (null == this.atomicGroupStateProviderEx)
{
AppTrace.TraceSource.WriteInfo("StatefulServiceReplica.Open", "{0} creating replicator", this.ToString());
//
// v1 replicator.
//
replicator = this.servicePartition.CreateReplicator(this.stateProvider, replicatorSettings);
this.stateReplicator = replicator.StateReplicator;
this.atomicGroupStateReplicator = this.stateReplicator as IAtomicGroupStateReplicator;
}
else
{
AppTrace.TraceSource.WriteInfo("StatefulServiceReplica.Open", "{0} creating atomic group replicator", this.ToString());
//
// v2 replicator.
//
replicatorEx = this.servicePartitionEx.CreateReplicatorEx(this.atomicGroupStateProviderEx, replicatorSettings, replicatorLogSettings);
this.atomicGroupStateReplicatorEx = replicatorEx.StateReplicator;
}
//
// Perform local open functionality. Initialize and open state provider broker.
//
this.stateProviderBroker.Initialize(this.initializationParameters);
await this.stateProviderBroker.OpenAsync(
openMode,
this.servicePartitionEx,
this.atomicGroupStateReplicatorEx,
cancellationToken);
//
// Perform custom open functionality.
//
await this.OnOpenAsync(openMode, cancellationToken);
//
// Change current replica state.
//
this.replicaState = ReplicaState.Opened;
AppTrace.TraceSource.WriteNoise("StatefulServiceReplica.State", "{0} is {1}", this.ToString(), this.replicaState);
//
// Done.
//
return((null != replicator) ? replicator as IReplicator : replicatorEx as IReplicator);
}
