private async Task <bool> ReportLoadOnChangeRole(ReplicaRole role)
{
MetricReliableStateManager target = new MetricReliableStateManager(
this.GetContext(),
new JsonReliableStateSerializerResolver(),
this.GetConfig(),
new MockReliableStateManager());
ManualResetEvent reset = new ManualResetEvent(false);
bool actual = false;
MockStatefulServicePartition partition = new MockStatefulServicePartition()
{
OnReportLoad = (metrics) =>
{
actual = true;
reset.Set();
}
};
await((IStateProviderReplica)target).OpenAsync(ReplicaOpenMode.New, partition, CancellationToken.None);
await((IStateProviderReplica)target).ChangeRoleAsync(role, CancellationToken.None);
// this may yield false negatives because we're at the mercy of the task scheduler
// to actually execute the reporting task in a timely manner, which depends on external factors.
reset.WaitOne(TimeSpan.FromSeconds(10));
return(actual);
}
public void Initialize(StatefulServiceInitializationParameters initializationParameters)
{
this.currentRole = ReplicaRole.Unknown;
this.initializationParameters = initializationParameters;
this.TraceInfo("IStatefulServiceReplica::Initialize invoked for service {0}.", this.initializationParameters.ServiceName);
}
private async Task ChangeRoleAsync(ReplicaRole newRole)
{
ReplicaRole = newRole;
await _serviceInstance.InvokeOnChangeRoleAsync(newRole, ChangeRoleCancellation.Token);
await _stateManager.ChangeRoleAsync(newRole, ChangeRoleCancellation.Token);
}
protected override Task OnChangeRoleAsync(ReplicaRole newRole, CancellationToken cancellationToken)
{
ServiceEventSource.Current.ServiceMessage(this, "*** OnChangeRole for {0}, Parition {1}. new role is: {2}", this.Context.ServiceName, this.Context.PartitionId, newRole.ToString());
// TODO: record role change
return(base.OnChangeRoleAsync(newRole, cancellationToken));
}
/// <summary>
/// <see cref="ReliableCollectionsActorStateProvider"/> is currently in PREVIEW.
/// Initializes a new instance of the ReliableDictionaryActorStateProvider class
/// with specified configuration.
/// </summary>
/// <param name="stateManagerConfig">
/// A <see cref="ReliableStateManagerConfiguration"/> that describes <see cref="IReliableStateManager"/> configuration.
/// </param>
/// <param name="actorStateDictionaryCount">
/// Number of <see cref="IReliableDictionary{TKey, TValue}"/> across which actor states will be partitioned and stored.
/// </param>
/// <param name="reminderDictionaryCount">
/// Number of <see cref="IReliableDictionary{TKey, TValue}"/> across which reminders will be partitioned and stored.
/// </param>
/// <remarks>
/// Values for <paramref name="actorStateDictionaryCount"/> and <paramref name="reminderDictionaryCount"/> can be specified
/// only once when the Actor Service is created for first time. It cannot be changed after that and
/// <see cref="ReliableCollectionsActorStateProvider"/> will ignore any values that are different from first time.
/// </remarks>
public ReliableCollectionsActorStateProvider(
ReliableStateManagerConfiguration stateManagerConfig,
int actorStateDictionaryCount,
int reminderDictionaryCount)
{
if (actorStateDictionaryCount < 1)
{
throw new ArgumentException("Value for actorStateDictionaryCount cannot be less than 1.");
}
if (reminderDictionaryCount < 1)
{
throw new ArgumentException("Value for reminderDictionaryCount cannot be less than 1.");
}
this.traceId = string.Empty;
this.isLogicalTimeManagerInitialized = false;
this.isDictionariesInitialized = false;
this.replicaRole = ReplicaRole.Unknown;
this.transientErrorRetryDelay = TimeSpan.FromSeconds(DefaultTransientErrorRetryDelaySeconds);
this.userDefinedStateManagerConfig = stateManagerConfig;
this.userDefinedActorStateDictionaryCount = actorStateDictionaryCount;
this.userDefinedReminderDictionaryCount = reminderDictionaryCount;
this.logicalTimeManager = new VolatileLogicalTimeManager(this);
this.actorStateSerializer = new ActorStateProviderSerializer();
this.stateProviderHelper = new ActorStateProviderHelper(this);
}
/// <inheritdoc />
public Task ChangeRoleAsync(ReplicaRole newRole, CancellationToken cancellationToken)
{
this.Log("New role: {0}", newRole);
switch (newRole)
{
case ReplicaRole.Unknown:
throw new InvalidOperationException("Unexpected ChangeRole");
case ReplicaRole.Primary:
this.cancellationTokenSource = new CancellationTokenSource();
break;
case ReplicaRole.None:
this.StopProcessing();
break;
case ReplicaRole.IdleSecondary:
this.StartCopyAndReplicationPump();
this.StopProcessing();
break;
case ReplicaRole.ActiveSecondary:
this.StartReplicationPump();
this.StopProcessing();
break;
}
this.currentRole = newRole;
return(Task.FromResult(this.listenerUri.ToString()));
}
/// <summary>
/// Called on the replica when the role is changing.
/// </summary>
/// <param name="newRole">New role for the replica.</param>
/// <param name="cancellationToken">Propagates notification that operation should be canceled.</param>
/// <returns></returns>
public virtual async Task <string> ChangeRoleAsync(ReplicaRole newRole, CancellationToken cancellationToken)
{
ReplicaRole currentRole = this.replicaRole;
AppTrace.TraceSource.WriteNoise("StatefulServiceReplica.ChangeRole", "{0} starting role change from {1} to {2}", this.ToString(), currentRole, newRole);
//
// Perform local change role functionality.Change role in the state provider broker.
//
await this.stateProviderBroker.ChangeRoleAsync(newRole, cancellationToken);
//
// Perform custom change role functionality.
//
await this.OnChangeRoleAsync(newRole, cancellationToken);
//
// Retrieve endpoint.
//
string endpoint = await this.GetEndpointAsync(newRole, cancellationToken);
AppTrace.TraceSource.WriteNoise("StatefulServiceReplica.ChangeRole", "{0} completed role change from {1} to {2}", this.ToString(), currentRole, newRole);
this.replicaRole = newRole;
return(endpoint);
}
/// <summary>
/// Overrides <see cref="StatefulServiceBase.OnChangeRoleAsync(ReplicaRole, CancellationToken)"/>.
/// </summary>
/// <param name="newRole">The new role for the replica.</param>
/// <param name="cancellationToken">The token to monitor for cancellation requests.</param>
/// <returns>A task that represents the asynchronous operation performed when the replica becomes primary.</returns>
protected override async Task OnChangeRoleAsync(ReplicaRole newRole, CancellationToken cancellationToken)
{
ActorTrace.Source.WriteInfoWithId(TraceType, this.Context.TraceId, "Begin change role. New role: {0}.",
newRole);
if (newRole == ReplicaRole.Primary)
{
this.actorManagerAdapter.ActorManager = new ActorManager(this);
await this.actorManagerAdapter.OpenAsync(this.Partition, cancellationToken);
this.ActorManager.DiagnosticsEventManager.ActorChangeRole(this.replicaRole, newRole);
}
else
{
if ((this.ActorManager != null) && (this.ActorManager.DiagnosticsEventManager != null))
{
this.ActorManager.DiagnosticsEventManager.ActorChangeRole(this.replicaRole, newRole);
}
await this.actorManagerAdapter.CloseAsync(cancellationToken);
}
this.replicaRole = newRole;
ActorTrace.Source.WriteInfoWithId(TraceType, this.Context.TraceId, "End change role. New role: {0}.",
newRole);
}
internal OperationFetcher(IOperationQueue copyOperationQueue, IOperationQueue replicationOperationQueue, bool drainQueuesInParallel)
{
this.copyOperationQueue = copyOperationQueue;
this.replicationOperationQueue = replicationOperationQueue;
this.drainQueuesInParallel = drainQueuesInParallel;
this.role = ReplicaRole.None;
}
public OperationFetcher(IStateReplicator stateReplicator, OperationProcessorInfo copyOperationProcessor, OperationProcessorInfo replicationOperationProcessor, bool drainQueuesInParallel)
{
Requires.Argument("partition", stateReplicator).NotNull();
Requires.Argument("copyOperationProcessor", copyOperationProcessor).NotNull();
Requires.Argument("replicationOperationProcessor", replicationOperationProcessor).NotNull();
if (copyOperationProcessor.Callback == null)
{
throw new ArgumentException("copyOperationProcessor.Callback cannot be null");
}
if (replicationOperationProcessor.Callback == null)
{
throw new ArgumentException("replicationOperationProcessor.Callback cannot be null");
}
this.copyOperationQueue = new OperationQueue(() => stateReplicator.GetCopyStream(), copyOperationProcessor)
{
Name = "CopyQueue"
};
this.replicationOperationQueue = new OperationQueue(() => stateReplicator.GetReplicationStream(), replicationOperationProcessor)
{
Name = "ReplicationQueue"
};
this.drainQueuesInParallel = drainQueuesInParallel;
this.role = ReplicaRole.None;
}
public async Task AddReplicaAsync(ReplicaRole role, long?replicaId = null, int activationDelayMs = 0)
{
var serviceContext = MockStatefulServiceContextFactory.Create(CodePackageActivationContext, ServiceTypeName, ServiceUri, Guid.NewGuid(), replicaId ?? _random.Next());
var replica = new MockStatefulServiceReplica <TStatefulService>(_serviceFactory, serviceContext, _stateManager);
await replica.CreateAsync(role);
_replicas.Add(replica);
}
private Task ChangeRoleAsync(IntPtr nativeEpoch, NativeTypes.FABRIC_REPLICA_ROLE nativeRole, CancellationToken cancellationToken)
{
Epoch epoch = Epoch.FromNative(nativeEpoch);
ReplicaRole replicaRole = (ReplicaRole)nativeRole;
return(this.replicator.ChangeRoleAsync(epoch, replicaRole, cancellationToken));
}
protected override async Task OnChangeRoleAsync(
ReplicaRole newRole,
CancellationToken cancellationToken)
{
var payload = new StatefulServiceEventPayloadOnChangeRole(newRole);
await this.serviceEvents.NotifyChangeRoleAsync(payload, cancellationToken);
}
private Task ChangeRoleAsyncHelper(Epoch epoch, ReplicaRole role, CancellationToken cancellationToken)
{
return(Utility.WrapNativeAsyncInvoke(
(callback) => this.ChangeRoleBeginWrapper(epoch, role, callback),
this.ChangeRoleEndWrapper,
cancellationToken,
"FabricReplicator.ChangeRole"));
}
private void RoleChangeTest(ReplicaRole expectedRole, params object[] roleChangeArgs)
{
var inst = OperationFetcherTest.Create();
inst.DoStateChangeOperations(roleChangeArgs);
Assert.AreEqual <ReplicaRole>(expectedRole, inst.Fetcher.Role);
}
async Task IStateProviderReplica.ChangeRoleAsync(ReplicaRole newRole, CancellationToken cancellationToken)
{
await((IStateProviderReplica)this._replica).ChangeRoleAsync(newRole, cancellationToken);
if (this._backupRestoreManager != null)
{
await this._backupRestoreManager.ChangeRoleAsync(newRole, cancellationToken);
}
}
public void ChangeRole(ReplicaRole newRole)
{
//// AppTrace.TraceMsg(TraceLogEventType.Information, "OperationFetcher.ChangeRole", "ChangeRole from {0} to {1}", this.role, newRole);
//// what are the state transistions possible?
ReplicaRole oldRole = this.role;
this.role = newRole;
if (oldRole == ReplicaRole.None && newRole == ReplicaRole.IdleSecondary)
{
if (this.drainQueuesInParallel)
{
OperationFetcher.DrainQueueAsyncHelper(this.copyOperationQueue);
OperationFetcher.DrainQueueAsyncHelper(this.replicationOperationQueue);
}
else
{
OperationFetcher.DrainQueueAsyncHelper(this.copyOperationQueue).ContinueWith(copyQueueDrainTask =>
{
//// AppTrace.TraceMsg(TraceLogEventType.Verbose, "OperationFetcher.QueueDrain", "Starting replication queue");
if (this.role == ReplicaRole.None)
{
//// AppTrace.TraceMsg(TraceLogEventType.Information, "OperationFetcher.QueueDrain", "ReplicaRole is None so not draining replication queue");
return;
}
OperationFetcher.DrainQueueAsyncHelper(this.replicationOperationQueue);
});
}
}
else if (oldRole == ReplicaRole.IdleSecondary && newRole == ReplicaRole.ActiveSecondary)
{
// No-Op
}
else if (oldRole == ReplicaRole.ActiveSecondary && newRole == ReplicaRole.Primary)
{
// No-Op
}
else if (oldRole == ReplicaRole.Primary && newRole == ReplicaRole.ActiveSecondary)
{
OperationFetcher.DrainQueueAsyncHelper(this.replicationOperationQueue);
}
else if (oldRole == ReplicaRole.None && newRole == ReplicaRole.Primary)
{
// No-op
}
else if (newRole == ReplicaRole.None)
{
// No-Op
}
else
{
// AppTrace.TraceMsg(TraceLogEventType.Error, "OperationFetcher.ChangeRole", "Invalid state change");
ReleaseAssert.Failfast(string.Format(CultureInfo.InvariantCulture, "Invalid State Change from {0} to {1}", oldRole, newRole));
}
}
/// <summary>
/// Handles node's role change.
/// </summary>
/// <param name="newRole">New <see cref="ReplicaRole" /> for this service replica.</param>
/// <param name="cancellationToken">Cancellation token to monitor for cancellation requests.</param>
/// <returns>
/// A <see cref="Task" /> that represents outstanding operation.
/// </returns>
public async Task OnChangeRoleAsync(ReplicaRole newRole, CancellationToken cancellationToken)
{
ServiceFabricProviderEventSource.Tracing.LogFabricServiceInformation(this.statefulService, $"TaskHubProxyListener OnChangeRoleAsync, current role = {this.currentRole}, new role = {newRole}");
if (newRole != ReplicaRole.Primary && this.currentRole == ReplicaRole.Primary)
{
await StopAsync();
}
this.currentRole = newRole;
ServiceFabricProviderEventSource.Tracing.LogFabricServiceInformation(this.statefulService, $"TaskHubProxyListener OnChangeRoleAsync, current role = {this.currentRole}");
}
protected override Task OnChangeRoleAsync(ReplicaRole newRole, CancellationToken cancellationToken)
{
if (newRole != ReplicaRole.Primary)
{
BackupRestoreTrace.TraceSource.WriteInfo(TraceType, "Stoping the WorkItem Handler");
WorkItemHandler.StopWorkItemHandler();
BackupRestoreTrace.TraceSource.WriteInfo(TraceType, "Stopped the WorkItem Handler");
}
return(base.OnChangeRoleAsync(newRole, cancellationToken));
}
private async Task ChangeRoleAsync(ReplicaRole newRole)
{
ReplicaRole = newRole;
await _serviceInstance.InvokeOnChangeRoleAsync(newRole, ChangeRoleCancellation.Token);
if (_serviceInstance.StateManager is MockReliableStateManager)
{
await((MockReliableStateManager)_serviceInstance.StateManager).ChangeRoleAsync(newRole, ChangeRoleCancellation.Token);
}
}
async Task IStateProviderReplica.ChangeRoleAsync(ReplicaRole newRole, CancellationToken cancellationToken)
{
await storeReplica.ChangeRoleAsync(newRole, cancellationToken);
//if (newRole == ReplicaRole.Primary)
// logicalTimeManager.Start();
//else
// logicalTimeManager.Stop();
replicaRole = newRole;
}
/// <summary>
/// Handles node's role change.
/// </summary>
/// <param name="newRole">New <see cref="ReplicaRole" /> for this service replica.</param>
/// <param name="cancellationToken">Cancellation token to monitor for cancellation requests.</param>
/// <returns>
/// A <see cref="Task" /> that represents outstanding operation.
/// </returns>
protected override async Task OnChangeRoleAsync(ReplicaRole newRole, CancellationToken cancellationToken)
{
ServiceFabricProviderEventSource.Tracing.LogFabricServiceInformation(this, $"Fabric On Change Role Async, current role = {this.currentRole}, new role = {newRole}");
foreach (var listener in this.serviceListeners)
{
await listener.OnChangeRoleAsync(newRole, cancellationToken);
}
this.currentRole = newRole;
ServiceFabricProviderEventSource.Tracing.LogFabricServiceInformation(this, $"Fabric On Change Role Async, current role = {this.currentRole}");
}
/// <summary>
/// Executed when the actor changes role.
/// </summary>
/// <param name="newRole"></param>
/// <param name="cancellationToken"></param>
/// <returns></returns>
protected override async Task OnChangeRoleAsync(ReplicaRole newRole, CancellationToken cancellationToken)
{
await base.OnChangeRoleAsync(newRole, cancellationToken);
// zero out load
if (newRole != ReplicaRole.Primary)
{
Partition.ReportLoad(new[] { new LoadMetric("AspNetStateService.ActiveSessionCount", 0) });
}
}
protected override async Task OnChangeRoleAsync(ReplicaRole newRole, CancellationToken cancellationToken)
{
ActorEventSource.Current.Message(
$"Start OnChangeRoleAsync: newRole={newRole}, NodeName={Context.NodeContext.NodeName}, ReplicaId={Context.ReplicaId}, PartitionId={Context.PartitionId}");
await base.OnChangeRoleAsync(newRole, cancellationToken);
ActorEventSource.Current.Message(
$"Stop OnChangeRoleAsync: newRole={newRole}, NodeName={Context.NodeContext.NodeName}, ReplicaId={Context.ReplicaId}, PartitionId={Context.PartitionId}");
}
/// <summary>
/// Return the endpoint for the fabric client to resolve. Depeding on the replica role
/// the endpoint may or may not be empty.
/// </summary>
/// <param name="role">Replica role used to determine if an endpoint should be opened.</param>
/// <param name="cancellationToken">Propagates notification that operations should be canceled.</param>
/// <returns></returns>
protected override Task <string> GetEndpointAsync(ReplicaRole role, CancellationToken cancellationToken)
{
AppTrace.TraceSource.WriteNoise("StatefulServiceReplica.GetEndpoint", "{0} role {1}", this.ToString(), role);
//
// No endpoint returned.
//
var tcs = new TaskCompletionSource <string>();
tcs.SetResult(string.Empty);
return(tcs.Task);
}
protected override Task OnChangeRoleAsync(ReplicaRole newRole, CancellationToken cancellationToken)
{
if (_bus != null && newRole != ReplicaRole.Primary)
{
_bus?.Dispose();
_bus = null;
Log.Information("#_#_Resetting local cluster");
}
return(base.OnChangeRoleAsync(newRole, cancellationToken));
}
/// <summary>
/// Invokes OnChangeRoleAsync on the provided <paramref name="service"/>.
/// </summary>
/// <param name="service"></param>
/// <param name="newRole"></param>
/// <param name="cancellationToken"></param>
/// <returns></returns>
public static Task InvokeOnChangeRoleAsync(this StatefulServiceBase service, ReplicaRole newRole, CancellationToken?cancellationToken = null)
{
if (service == null)
{
throw new ArgumentNullException(nameof(service));
}
//protected virtual Task RunAsync(CancellationToken cancellationToken)
var method = FindMethodInfo(service, "OnChangeRoleAsync");
return((Task)method.Invoke(service, new object[] { newRole, cancellationToken ?? CancellationToken.None }));
}
/// <summary>
/// <para>Initializes a new instance of the <see cref="System.Fabric.ReconfigurationInformation" /> class.</para>
/// </summary>
/// <param name="previousConfigurationRole"></param>
/// <param name="reconfigurationPhase"></param>
/// <param name="reconfigurationType"></param>
/// <param name="reconfigurationStartTimeUtc"></param>
public ReconfigurationInformation(
ReplicaRole previousConfigurationRole,
ReconfigurationPhase reconfigurationPhase,
ReconfigurationType reconfigurationType,
DateTime reconfigurationStartTimeUtc)
{
this.PreviousConfigurationRole = previousConfigurationRole;
this.ReconfigurationPhase = reconfigurationPhase;
this.ReconfigurationType = reconfigurationType;
this.ReconfigurationStartTimeUtc = reconfigurationStartTimeUtc;
}
internal void ActorChangeRole(ReplicaRole currentRole, ReplicaRole newRole)
{
var callbacks = this.OnActorChangeRole;
if (null != callbacks)
{
this.changeRoleDiagnosticData.CurrentRole = currentRole;
this.changeRoleDiagnosticData.NewRole = newRole;
callbacks(this.changeRoleDiagnosticData);
}
}
internal static void Assert(bool condition, string format, ReplicaRole param1)
{
if (condition == false)
{
var failFastMessage = string.Format(System.Globalization.CultureInfo.InvariantCulture, format, param1);
FailFast(failFastMessage);
// AMW - Force break into debugger for ease of debugging
Debugger.Break();
}
}
