public async Task ClusterOneTimeSetUp()
{
// create remote client and cluster
RcClient = await ConnectToRemoteControllerAsync().CfAwait();
RcCluster = await RcClient.CreateClusterAsync(RcClusterConfiguration).CfAwait();
}
/// <summary>
/// Adds a member to the cluster.
/// </summary>
/// <returns>The added member.</returns>
protected async Task <Member> AddMember()
{
var member = await RcClient.StartMemberAsync(RcCluster);
RcMembers[Guid.Parse(member.Uuid)] = member;
return(member);
}
/// <summary>
/// Removes a member from the cluster.
/// </summary>
/// <param name="memberId">The identifier of the member to remove.</param>
/// <returns>A task that will complete when the member has been removed.</returns>
protected async Task RemoveMember(Guid memberId)
{
if (RcMembers.TryRemove(memberId, out var member))
{
await RcClient.StopMemberAsync(RcCluster, member);
}
}
public async Task OneTimeTearDown()
{
// remove temp files
foreach (var fileInfo in new DirectoryInfo(Path.GetTempPath()).GetFiles())
{
try { fileInfo.Delete(); }
catch (Exception) { /* ignore */ }
}
// terminate & remove member (just in case)
if (RcMember != null)
{
await RcClient.StopMemberAsync(RcCluster, RcMember);
RcMember = null;
}
// terminate & remove client (needed) and cluster (just in case)
if (RcClient != null)
{
if (RcCluster != null)
{
await RcClient.ShutdownClusterAsync(RcCluster).CfAwait();
RcCluster = null;
}
await RcClient.ExitAsync().CfAwait();
RcClient = null;
}
}
public async Task MembersOneTimeTearDown()
{
// terminate & remove members
foreach (var member in RcMembers.Values)
{
await RcClient.StopMemberAsync(RcCluster, member);
}
}
public async Task MemberOneTimeTearDown()
{
// terminate & remove member
if (RcMember != null)
{
await RcClient.StopMemberAsync(RcCluster, RcMember);
}
}
public async Task ClusterOneTimeTearDown()
{
// terminate & remove client and cluster
if (RcClient != null)
{
if (RcCluster != null)
{
await RcClient.ShutdownClusterAsync(RcCluster).CfAwait();
}
await RcClient.ExitAsync().CfAwait();
}
}
public async Task TestPartitionsUpdatedAfterNewNode()
{
var cluster = ((HazelcastClient)Client).Cluster;
var partitionCount = cluster.Partitioner.Count;
Assert.AreEqual(271, partitionCount);
// 1 member = 1 partition owner
// partition table must update eventually
await AssertEx.SucceedsEventually(() =>
{
Assert.That(GetPartitionOwnerCount(cluster.Partitioner), Is.EqualTo(1));
},
4000, 500);
// add a second member
// make sure we wait for it to be added - otherwise we get our 2 partitions
// soon enough and then we stop the member before all migrations have run,
// and that confuses the server which then does weird things and never fully
// remove the second member, thus breaking the test.
var member2 = await RcClient.StartMemberWaitAddedAsync(Client, RcCluster, 2);
try
{
// partition table must update eventually
await AssertEx.SucceedsEventually(() =>
{
Assert.That(GetPartitionOwnerCount(cluster.Partitioner), Is.EqualTo(2));
},
4000, 500);
}
finally
{
// whatever happens, make sure the member is stopped
await RcClient.StopMemberWaitRemovedAsync(Client, RcCluster, member2);
}
// partition table must update eventually
await AssertEx.SucceedsEventually(() =>
{
Assert.That(GetPartitionOwnerCount(cluster.Partitioner), Is.EqualTo(1));
},
4000, 500);
}
public async Task TearDown()
{
// terminate & remove member
if (RcMember != null)
{
await RcClient.StopMemberAsync(RcCluster, RcMember);
RcMember = null;
}
// terminate & remove cluster
if (RcCluster != null)
{
await RcClient.ShutdownClusterAsync(RcCluster).CfAwait();
RcCluster = null;
}
}
protected async ValueTask <IHazelcastClient> StartClientAsync(string serverXml, bool enableSsl, bool?validateCertificateChain,
bool?validateCertificateName, bool?checkCertificateRevocation, string certSubjectName, byte[] clientCertificate,
string certPassword, bool failFast = false)
{
RcCluster = await RcClient.CreateClusterAsync(serverXml);
RcMember = await RcClient.StartMemberAsync(RcCluster);
var options = HazelcastOptions.Build();
options.Networking.Addresses.Clear();
//options.Networking.Addresses.Add("localhost:5701");
options.Networking.Addresses.Add("127.0.0.1:5701");
((IClusterOptions)options).ClusterName = RcCluster.Id;
options.LoggerFactory.Creator = () => LoggerFactory;
var sslOptions = options.Networking.Ssl;
sslOptions.Enabled = enableSsl;
sslOptions.ValidateCertificateChain = validateCertificateChain ?? sslOptions.ValidateCertificateChain;
sslOptions.ValidateCertificateName = validateCertificateName ?? sslOptions.ValidateCertificateName;
sslOptions.CertificateName = certSubjectName ?? sslOptions.CertificateName;
sslOptions.CheckCertificateRevocation = checkCertificateRevocation ?? sslOptions.CheckCertificateRevocation;
if (failFast)
{
// default value is 20s but if we know we are going to fail, no point trying again and again
options.Networking.ConnectionRetry.ClusterConnectionTimeoutMilliseconds = 2_000;
}
if (enableSsl && clientCertificate != null)
{
var certFilePath = CreateTmpFile(clientCertificate);
sslOptions.CertificatePath = certFilePath;
if (certPassword != null)
{
sslOptions.CertificatePassword = certPassword;
}
}
return(await HazelcastClientFactory.StartNewClientAsync(options));
}
private async Task <Response> ExecuteRemotePythonAsync(string script)
{
return(await RcClient.ExecuteOnControllerAsync(RcCluster.Id, script, Lang.PYTHON));
}
public async Task MemberOneTimeSetUp()
{
// add a member to the cluster
RcMember = await RcClient.StartMemberAsync(RcCluster);
}
public async Task ReconnectAsync()
{
using var _ = HConsoleForTest();
// add one member
var member = await RcClient.StartMemberAsync(RcCluster);
// connect & use a client
var options = new HazelcastOptionsBuilder()
.With((configuration, o) =>
{
o.ClusterName = RcCluster.Id;
o.Networking.Addresses.Clear();
o.Networking.Addresses.Add("127.0.0.1:5701");
o.Networking.ReconnectMode = ReconnectMode.ReconnectAsync;
o.Networking.ConnectionRetry.InitialBackoffMilliseconds = 1_000; // initially wait for 1s
o.Networking.ConnectionRetry.Multiplier = 1; // and keep waiting for 1s
o.Networking.ConnectionRetry.Jitter = 0; // exactly
o.Networking.ConnectionRetry.ClusterConnectionTimeoutMilliseconds = 240_000; // give up after 4mn
o.AddSubscriber(events => events
.StateChanged((sender, args) =>
{
HConsole.WriteLine(this, $"client state changed: {args.State}");
o.LoggerFactory.Service.CreateLogger <ReconnectTests>().LogDebug("Client state changed: {State}", args.State);
}));
})
.WithHConsoleLogger()
.WithUserSecrets(GetType().Assembly)
.Build();
HConsole.WriteLine(this, "Start client");
var client = await HazelcastClientFactory.StartNewClientAsync(options);
var map = await client.GetMapAsync <string, string>("test-map");
await map.SetAsync("key", "value");
Assert.That(await map.GetAsync("key"), Is.EqualTo("value"));
// kill the member
HConsole.WriteLine(this, "Stop member");
await RcClient.StopMemberWaitClosedAsync(client, RcCluster, member);
// using the client throws
// and the client is frantically trying to reconnect
HConsole.WriteLine(this, "Use client");
await AssertEx.ThrowsAsync <ClientOfflineException>(async() =>
{
await map.GetAsync("key");
});
// add a member again
// at some point, the client will reconnect
HConsole.WriteLine(this, "Start member");
member = await RcClient.StartMemberAsync(RcCluster);
// use the client
// initially, it should keep throwing, but eventually it should work
var stopwatch = System.Diagnostics.Stopwatch.StartNew();
var count = 0;
HConsole.WriteLine(this, "Use client");
await AssertEx.SucceedsEventually(async() =>
{
HConsole.WriteLine(this, $"Attempt {count++} at {stopwatch.ElapsedMilliseconds}ms");
// of course the value will be gone, but after a while this should not throw
Assert.That(await map.GetAsync("key"), Is.Null);
}, 30_000, 500);
// we're done
}
