private void UpdateReplicatorSettingsHelper(ReplicatorSettings replicatorSettings)
{
// Initialize replicator
if (replicatorSettings == null)
{
AppTrace.TraceSource.WriteNoise("FabricReplicator.UpdateReplicatorSettings", "Using default replicator settings");
}
else
{
AppTrace.TraceSource.WriteNoise("FabricReplicator.UpdateReplicatorSettings", "Replicator Settings - address {0}, listenAddress {1}, publishAddress {2}, retyrInterval {3}, ackInterval {4}, credentials provided {5}", replicatorSettings.ReplicatorAddress, replicatorSettings.ReplicatorListenAddress, replicatorSettings.ReplicatorPublishAddress, replicatorSettings.RetryInterval, replicatorSettings.BatchAcknowledgementInterval, replicatorSettings.SecurityCredentials != null);
}
using (var pin = new PinCollection())
{
unsafe
{
if (replicatorSettings == null)
{
this.nativeStateReplicator.UpdateReplicatorSettings(IntPtr.Zero);
}
else
{
IntPtr nativeReplicatorSettings = replicatorSettings.ToNative(pin);
this.nativeStateReplicator.UpdateReplicatorSettings(nativeReplicatorSettings);
}
}
}
}
public async Task <IReplicator> OpenAsync(
ReplicaOpenMode openMode,
IStatefulServicePartition servicePartition,
CancellationToken cancellationToken)
{
var self = this.serviceParameters;
this.logger = new Logger(self)
{
Prefix = () => $"[{this.unsafeRole}] "
};
this.logger.Log("OpenAsync");
IReplicator result;
StateProvider provider;
lock (this.replicaLock)
{
this.partition = servicePartition;
provider = this.stateProvider = new StateProvider(this.logFilePath, this.logger, this.serializer);
var replicatorSettings = ReplicatorSettings.LoadFrom(
self.CodePackageActivationContext,
ConfigPackageName,
ReplicatorConfigSectionName);
replicatorSettings.BatchAcknowledgementInterval = TimeSpan.FromMilliseconds(1);
result = this.fabricReplicator = servicePartition.CreateReplicator(this.stateProvider, replicatorSettings);
}
await provider.Initialize();
this.logger.Log("Completed OpenAsync");
return(result);
}
public ReplicatorResiliencySpec(ITestOutputHelper helper) : base(SpecConfig, helper)
{
_sys1 = Sys;
_sys3 = ActorSystem.Create(Sys.Name, Sys.Settings.Config);
_sys2 = ActorSystem.Create(Sys.Name, Sys.Settings.Config);
var settings = ReplicatorSettings.Create(Sys)
.WithGossipInterval(TimeSpan.FromSeconds(1.0))
.WithMaxDeltaElements(10)
.WithRestartReplicatorOnFailure(true);
var props = BackoffSupervisor.Props(
Backoff.OnStop(
childProps: Replicator.Props(settings),
childName: "replicator",
minBackoff: TimeSpan.FromSeconds(3),
maxBackoff: TimeSpan.FromSeconds(300),
randomFactor: 0.2,
maxNrOfRetries: -1)
.WithFinalStopMessage(m => m is Terminate))
.WithDeploy(Deploy.Local).WithDispatcher(settings.Dispatcher);
_replicator1 = _sys1.ActorOf(props, "replicatorSuper");
_replicator2 = _sys2.ActorOf(props, "replicatorSuper");
_replicator3 = _sys3.ActorOf(props, "replicatorSuper");
}
/// <summary>
/// This is used by Kvs and Volatile actor state provider.
/// </summary>
/// <param name="codePackage"></param>
/// <param name="actorImplType"></param>
/// <returns></returns>
internal static ReplicatorSettings GetActorReplicatorSettings(CodePackageActivationContext codePackage, Type actorImplType)
{
var settings = ReplicatorSettings.LoadFrom(
codePackage,
ActorNameFormat.GetConfigPackageName(actorImplType),
ActorNameFormat.GetFabricServiceReplicatorConfigSectionName(actorImplType));
settings.SecurityCredentials = SecurityCredentials.LoadFrom(
codePackage,
ActorNameFormat.GetConfigPackageName(actorImplType),
ActorNameFormat.GetFabricServiceReplicatorSecurityConfigSectionName(actorImplType));
var nodeContext = FabricRuntime.GetNodeContext();
var endpoint = codePackage.GetEndpoint(ActorNameFormat.GetFabricServiceReplicatorEndpointName(actorImplType));
settings.ReplicatorAddress = string.Format(
CultureInfo.InvariantCulture,
"{0}:{1}",
nodeContext.IPAddressOrFQDN,
endpoint.Port);
if (!settings.MaxPrimaryReplicationQueueSize.HasValue)
{
settings.MaxPrimaryReplicationQueueSize = DefaultMaxPrimaryReplicationQueueSize;
}
if (!settings.MaxSecondaryReplicationQueueSize.HasValue)
{
settings.MaxSecondaryReplicationQueueSize = DefaultMaxSecondaryReplicationQueueSize;
}
return(settings);
}
private static StatefulServiceReplicaBroker CreateDefaultBroker(ReplicatorSettings settings = null)
{
var serviceStub = new StatefulServiceStub();
((StatefulServiceStub)serviceStub).ReplicatorSettingsInternal = settings;
return(new StatefulServiceReplicaBroker(serviceStub, StatefulPartitionTest.DefaultInitializationParameters, StatefulPartitionTest.DefaultReplicaId));
}
private void UpdateReplicatorSettings(string endpoint)
{
if (this.replicator != null)
{
ReplicatorSettings replicatorSettings = this.CreateReplicatorSettings(endpoint);
this.replicator.StateReplicator.UpdateReplicatorSettings(replicatorSettings);
}
}
protected ReplicatorChaosSpec(ReplicatorChaosSpecConfig config) : base(config, typeof(ReplicatorChaosSpec))
{
_cluster = Akka.Cluster.Cluster.Get(Sys);
_timeout = Dilated(TimeSpan.FromSeconds(3));
_replicator = Sys.ActorOf(Replicator.Props(ReplicatorSettings.Create(Sys)
.WithRole("backend")
.WithGossipInterval(TimeSpan.FromSeconds(1))), "replicator");
}
private IActorRef StartReplicator(ActorSystem system)
{
return(system.ActorOf(Replicator.Props(
ReplicatorSettings.Create(system)
.WithGossipInterval(TimeSpan.FromSeconds(1))
.WithPruning(TimeSpan.FromSeconds(1), maxPruningDissemination)),
"replicator"));
}
public ReplicatorPruningSpec(ReplicatorPruningSpecConfig config) : base(config)
{
_cluster = Cluster.Cluster.Get(Sys);
_timeout = Dilated(TimeSpan.FromSeconds(3));
_replicator = Sys.ActorOf(Replicator.Props(ReplicatorSettings.Create(Sys)
.WithGossipInterval(TimeSpan.FromSeconds(1))
.WithPruning(pruningInterval: TimeSpan.FromSeconds(1), maxPruningDissemination: _maxPruningDissemination)),
"replicator");
}
// Used by KvsActorStateProvider_V2
//
public KeyValueStoreReplica_V2(
ReplicatorSettings replicatorSettings,
KeyValueStoreReplicaSettings_V2 kvsSettings)
: this(
"KVS_V2", // Unused ESE storeName
replicatorSettings,
kvsSettings)
{
}
private KeyValueStoreReplica_V2(
string storeName,
ReplicatorSettings replicatorSettings,
KeyValueStoreReplicaSettings_V2 kvsSettings)
: base(storeName, replicatorSettings)
{
Requires.Argument <KeyValueStoreReplicaSettings_V2>("kvsSettings", kvsSettings).NotNull();
this.KeyValueStoreReplicaSettings_V2 = kvsSettings;
}
private IActorRef NewReplicator(ActorSystem system = null)
{
if (system == null)
{
system = Sys;
}
return(system.ActorOf(Replicator.Props(
ReplicatorSettings.Create(system).WithGossipInterval(TimeSpan.FromSeconds(1))),
"replicator-" + testStepCounter));
}
public KeyValueStoreWrapper(string storeName, LocalStoreSettings storeSettings,
ReplicatorSettings replicatorSettings, KeyValueStoreReplicaSettings kvsReplicaSettings,
Action <KeyValueStoreEnumerator> copyHandler,
Action <IEnumerator <KeyValueStoreNotification> > replicationHandler,
Func <CancellationToken, Task <bool> > onDataLossCallback)
: base(storeName, storeSettings, replicatorSettings, kvsReplicaSettings)
{
this.copyHandler = copyHandler;
this.replicationHandler = replicationHandler;
this.onDataLossCallback = onDataLossCallback;
}
private ReplicatorSettings GetReplicatorSettings()
{
var replicatorSettings = ReplicatorSettings.LoadFrom(
(CodePackageActivationContext)this.serviceContext.ServiceContext.CodePackageActivationContext,
"Config",
"ReplicatorConfig");
////replicatorSettings.ReplicatorAddress
replicatorSettings.RequireServiceAck = true;
return(replicatorSettings);
}
public void Cluster_CRDT_should_support_prefer_oldest_members()
{
// disable gossip and delta replication to only verify the write and read operations
var oldestReplicator = Sys.ActorOf(
Replicator.Props(
ReplicatorSettings.Create(Sys).WithPreferOldest(true).WithGossipInterval(TimeSpan.FromMinutes(1))),//.withDeltaCrdtEnabled(false)),
"oldestReplicator");
Within(TimeSpan.FromSeconds(5), () =>
{
var countProbe = CreateTestProbe();
AwaitAssert(() =>
{
oldestReplicator.Tell(GetReplicaCount.Instance, countProbe.Ref);
countProbe.ExpectMsg(new ReplicaCount(3));
});
});
EnterBarrier("oldest-replicator-started");
var probe = CreateTestProbe();
RunOn(() =>
{
oldestReplicator.Tell(
Dsl.Update(KeyK, new LWWRegister <string>(Cluster.SelfUniqueAddress, "0"), _writeTwo, a => a.WithValue(Cluster.SelfUniqueAddress, "1")),
probe.Ref);
probe.ExpectMsg(new UpdateSuccess(KeyK, null));
}, _second);
EnterBarrier("updated-1");
RunOn(() =>
{
// replicated to oldest
oldestReplicator.Tell(new Get(KeyK, ReadLocal.Instance), probe.Ref);
var msg = probe.ExpectMsg <GetSuccess>(m => m.Data is LWWRegister <string>);
((LWWRegister <string>)msg.Data).Value.Should().Be("1");
//probe.ExpectMsg<GetSuccess[LWWRegister[String]]>.dataValue.value should === ("1");
}, _first);
RunOn(() =>
{
// not replicated to third (not among the two oldest)
oldestReplicator.Tell(Dsl.Get(KeyK, ReadLocal.Instance), probe.Ref);
probe.ExpectMsg(new NotFound(KeyK, null));
// read from oldest
oldestReplicator.Tell(Dsl.Get(KeyK, _readTwo), probe.Ref);
var msg = probe.ExpectMsg <GetSuccess>(m => m.Data is LWWRegister <string>);
((LWWRegister <string>)msg.Data).Value.Should().Be("1");
//probe.ExpectMsg<GetSuccess[LWWRegister[String]]>.dataValue.value should === ("1");
}, _third);
EnterBarrierAfterTestStep();
}
private ReplicatorSettings GetReplicatorSettingsHelper()
{
AppTrace.TraceSource.WriteNoise("FabricReplicator.GetReplicatorSettings", "Getting Replicator Settings from native replicator");
using (var pin = new PinCollection())
{
NativeRuntime.IFabricReplicatorSettingsResult replicatorSettingsResult =
nativeStateReplicator.GetReplicatorSettings();
return(ReplicatorSettings.CreateFromNative(replicatorSettingsResult));
}
}
private void ReplicatorSettingsTestHelper(
ReplicatorSettings valueToPass,
NativeTypes.FABRIC_REPLICATOR_SETTINGS_FLAGS flagsExpected,
Action <NativeTypes.FABRIC_REPLICATOR_SETTINGS> validator)
{
var rv = StatefulPartitionTest.ReplicatorCreationHelper(valueToPass);
NativeTypes.FABRIC_REPLICATOR_SETTINGS nativeObj = rv.Item1.ReplicatorSettings.Value;
Assert.AreEqual <uint>((uint)flagsExpected, rv.Item1.ReplicatorSettings.Value.Flags);
validator(nativeObj);
}
public void Durable_CRDT_should_stop_Replicator_if_Load_fails()
{
RunOn(() =>
{
var r = Sys.ActorOf(Replicator.Props(
ReplicatorSettings.Create(Sys).WithDurableStoreProps(TestDurableStoreProps(failLoad: true))),
"replicator-" + testStepCounter);
Watch(r);
ExpectTerminated(r);
}, first);
EnterBarrierAfterTestStep();
}
public Task <IReplicator> OpenAsync(ReplicaOpenMode openMode, IStatefulServicePartition partition, CancellationToken cancellationToken)
{
this.TraceInfo("IStatefulServiceReplica::OpenAsync invoked. Open mode: {0}.", openMode);
this.partition = partition;
ReplicatorSettings replicatorSettings = this.CreateReplicatorSettings();
this.replicator = partition.CreateReplicator(this, replicatorSettings);
this.TraceInfo("IStatefulServiceReplica::OpenAsync completed.");
return(Task.FromResult(this.replicator as IReplicator));
}
public void Durable_CRDT_should_reply_with_StoreFailure_if_store_fails()
{
RunOn(() =>
{
var r = Sys.ActorOf(Replicator.Props(
ReplicatorSettings.Create(Sys).WithDurableStoreProps(TestDurableStoreProps(failStore: true))),
"replicator-" + testStepCounter);
r.Tell(Dsl.Update(keyA, GCounter.Empty, WriteLocal.Instance, "a", c => c.Increment(cluster)));
ExpectMsg(new StoreFailure(keyA, "a"));
}, first);
EnterBarrierAfterTestStep();
}
private ReplicatorSettings GetReplicatorSettings(ClusterShardingSettings shardingSettings)
{
var configuredSettings = ReplicatorSettings.Create(Context.System.Settings.Config.GetConfig("akka.cluster.sharding.distributed-data"));
var settingsWithRoles = configuredSettings.WithRole(shardingSettings.Role);
if (shardingSettings.RememberEntities)
{
return(settingsWithRoles);
}
else
{
return(settingsWithRoles.WithDurableKeys(ImmutableHashSet <string> .Empty));
}
}
protected ReplicatorPruningSpec(ReplicatorPruningSpecConfig config) : base(config,
typeof(ReplicatorPruningSpec))
{
_cluster = Akka.Cluster.Cluster.Get(Sys);
_timeout = Dilated(TimeSpan.FromSeconds(3));
_replicator = Sys.ActorOf(Replicator.Props(ReplicatorSettings.Create(Sys)
.WithGossipInterval(TimeSpan.FromSeconds(1))
.WithPruning(pruningInterval: TimeSpan.FromSeconds(1),
maxPruningDissemination: _maxPruningDissemination)),
"replicator");
First = config.First;
Second = config.Second;
Third = config.Third;
}
internal KeyValueStateProvider(ReplicatorSettings replicatorSettings,
LocalStoreSettings localStoreSettings,
bool enableIncrementalBackup, int?logTruncationIntervalInMinutes)
{
userDefinedReplicatorSettings = replicatorSettings;
userDefinedLocalStoreSettings = localStoreSettings;
userDefinedEnableIncrementalBackup = enableIncrementalBackup;
userDefinedLogTruncationInterval = logTruncationIntervalInMinutes;
replicaRole = ReplicaRole.Unknown;
restoreSettings = new RestoreSettings(true);
isBackupInProgress = 0;
backupCallbackLock = new SemaphoreSlim(1);
backupCallbackCts = null;
backupCallbackTask = null;
isClosingOrAborting = false;
}
public void SettingsShouldContainProperDefaultValues()
{
var settings = ReplicatorSettings.Create(Sys);
settings.Role.ShouldBe(String.Empty);
settings.GossipInterval.ShouldBe(TimeSpan.FromSeconds(2));
settings.NotifySubscribersInterval.ShouldBe(TimeSpan.FromMilliseconds(500));
settings.MaxDeltaElements.ShouldBe(500);
settings.Dispatcher.ShouldBe("akka.actor.internal-dispatcher");
settings.PruningInterval.ShouldBe(TimeSpan.FromSeconds(120));
settings.MaxPruningDissemination.ShouldBe(TimeSpan.FromSeconds(300));
settings.PruningMarkerTimeToLive.ShouldBe(TimeSpan.FromHours(6));
settings.RestartReplicatorOnFailure.ShouldBeFalse();
settings.MaxDeltaSize.ShouldBe(50);
settings.DurableKeys.Count.ShouldBe(0);
settings.DurableStoreProps.ShouldBe(Props.Empty);
settings.DurablePruningMarkerTimeToLive.ShouldBe(TimeSpan.FromDays(10));
Sys.Settings.Config.GetTimeSpan("akka.cluster.distributed-data.serializer-cache-time-to-live")
.ShouldBe(TimeSpan.FromSeconds(10));
Sys.Settings.Config.GetString("akka.cluster.distributed-data.durable.store-actor-class")
.ShouldBe("Akka.DistributedData.LightningDB.LmdbDurableStore, Akka.DistributedData.LightningDB");
Sys.Settings.Config.GetString("akka.cluster.distributed-data.durable.use-dispatcher")
.ShouldBe("akka.cluster.distributed-data.durable.pinned-store");
Sys.Settings.Config.GetString("akka.actor.serializers.akka-data-replication")
.ShouldBe("Akka.DistributedData.Serialization.ReplicatorMessageSerializer, Akka.DistributedData");
Sys.Settings.Config.GetString("akka.actor.serializers.akka-replicated-data")
.ShouldBe("Akka.DistributedData.Serialization.ReplicatedDataSerializer, Akka.DistributedData");
Sys.Settings.Config.GetString("akka.actor.serialization-bindings.\"Akka.DistributedData.IReplicatorMessage, Akka.DistributedData\"")
.ShouldBe("akka-data-replication");
Sys.Settings.Config.GetString("akka.actor.serialization-bindings.\"Akka.DistributedData.IReplicatedDataSerialization, Akka.DistributedData\"")
.ShouldBe("akka-replicated-data");
Sys.Settings.Config.GetInt("akka.actor.serialization-identifiers.\"Akka.DistributedData.Serialization.ReplicatedDataSerializer, Akka.DistributedData\"")
.ShouldBe(11);
Sys.Settings.Config.GetInt("akka.actor.serialization-identifiers.\"Akka.DistributedData.Serialization.ReplicatorMessageSerializer, Akka.DistributedData\"")
.ShouldBe(12);
}
ReplicatorSettings BuildReplicatorSettings()
{
var codePackage = initParams.CodePackageActivationContext;
var replicatorSettings = ReplicatorSettings.LoadFrom(codePackage,
"Config",
"ReplicatorConfig");
replicatorSettings.SecurityCredentials = SecurityCredentials.LoadFrom(codePackage,
"Config",
"ReplicatorSecurityConfig");
var nodeContext = FabricRuntime.GetNodeContext();
var endpoint = codePackage.GetEndpoint("ReplicatorEndpoint");
replicatorSettings.ReplicatorAddress = string.Format(CultureInfo.InvariantCulture, "{0}:{1}",
nodeContext.IPAddressOrFQDN, endpoint.Port);
return(replicatorSettings);
}
public ReplicatorSpec(ReplicatorSpecConfig config)
: base(config)
{
_config = config;
_cluster = Cluster.Cluster.Get(Sys);
var settings = ReplicatorSettings.Create(Sys).WithGossipInterval(TimeSpan.FromSeconds(1.0)).WithMaxDeltaElements(10);
var props = Replicator.Props(settings);
_replicator = Sys.ActorOf(props, "replicator");
_timeOut = Dilated(TimeSpan.FromSeconds(2.0));
_writeTwo = new WriteTo(2, _timeOut);
_writeMajority = new WriteMajority(_timeOut);
_writeAll = new WriteAll(_timeOut);
_readTwo = new ReadFrom(2, _timeOut);
_readMajority = new ReadMajority(_timeOut);
_readAll = new ReadAll(_timeOut);
}
private ReplicatorSettings CreateReplicatorSettings()
{
// Although we don't replicate any state, we need to keep the copy stream
// and the replication stream drained at all times. This is necessary in
// order to unblock role changes. For these reason, we specify a valid
// replicator address in the settings.
ReplicatorSettings replicatorSettings = new ReplicatorSettings();
string ipaddressOrFqdn = FabricRuntime.GetNodeContext().IPAddressOrFQDN;
int replicatorPort = FabricRuntime.GetActivationContext().GetEndpoint("ReplicatorEndpoint").Port;
replicatorSettings.ReplicatorAddress = string.Format(
CultureInfo.InvariantCulture,
"{0}:{1}",
ipaddressOrFqdn,
replicatorPort);
this.TraceInfo("Replicator address is {0}", replicatorSettings.ReplicatorAddress);
return(replicatorSettings);
}
private ReplicatorSettings CreateReplicatorSettings(string endpoint)
{
ReplicatorSettings replicatorSettings = new ReplicatorSettings();
if (endpoint != null)
{
TraceType.WriteInfo(
"Replicator address = {0}, useClusterSecuritySettingsForReplicator = {1}",
endpoint,
this.useClusterSecuritySettingsForReplicator);
replicatorSettings.ReplicatorAddress = endpoint;
if (this.useClusterSecuritySettingsForReplicator)
{
replicatorSettings.SecurityCredentials = SecurityCredentials.LoadClusterSettings();
}
}
return(replicatorSettings);
}
////
// IStatefulServiceReplica
////
public Task <IReplicator> OpenAsync(ReplicaOpenMode openMode, IStatefulServicePartition partition, CancellationToken cancellationToken)
{
if (partition == null)
{
throw new ArgumentNullException("partition");
}
Trace.WriteInfo(TraceType, "OpenAsync: {0}, OpenMode={1}", this.serviceId, openMode);
ReplicatorSettings replicatorSettings = this.CreateReplicatorSettings(this.replicatorAddress);
this.replicator = partition.CreateReplicator(this, replicatorSettings);
this.partition = partition;
this.serviceEndpoint = this.infrastructureServiceAgent.RegisterInfrastructureService(
this.initializationParameters.PartitionId,
this.initializationParameters.ReplicaId,
this.coordinator);
return(Utility.CreateCompletedTask <IReplicator>(this.replicator));
}
protected ReplicatorSpec(ReplicatorSpecConfig config)
: base(config, typeof(ReplicatorSpec))
{
_config = config;
_first = config.First;
_second = config.Second;
_third = config.Third;
_cluster = Akka.Cluster.Cluster.Get(Sys);
var settings = ReplicatorSettings.Create(Sys)
.WithGossipInterval(TimeSpan.FromSeconds(1.0))
.WithMaxDeltaElements(10);
var props = Replicator.Props(settings);
_replicator = Sys.ActorOf(props, "replicator");
_timeOut = Dilated(TimeSpan.FromSeconds(3.0));
_writeTwo = new WriteTo(2, _timeOut);
_writeMajority = new WriteMajority(_timeOut);
_writeAll = new WriteAll(_timeOut);
_readTwo = new ReadFrom(2, _timeOut);
_readMajority = new ReadMajority(_timeOut);
_readAll = new ReadAll(_timeOut);
}
