private static async Task RunAsync(IHazelcastClient client, CancellationToken cancellationToken)
{
// 'await using' ensure that both the client and the map will be disposed before the method returns
await using var c = client;
await using var map = await client.GetMapAsync <string, int>("test-map");
// loop while not canceled
while (!cancellationToken.IsCancellationRequested)
{
// pretend to do some work
var i = await map.GetAsync("foo");
i += 1;
await map.SetAsync("foo", i);
Console.WriteLine(i);
try
{
await Task.Delay(1000, cancellationToken);
}
catch (OperationCanceledException)
{
// expected
}
}
await client.DestroyAsync(map);
}
private static async Task RunTask(IHazelcastClient client, CancellationToken token, int id, int entryCount, ILogger logger)
{
logger.LogInformation($"Thread {id}: start");
IHMap <string, string> map = null;
while (!token.IsCancellationRequested)
{
try
{
map = await client.GetMapAsync <string, string>("soak1").ConfigureAwait(false);
break;
}
catch (Exception ex)
{
logger.LogError($"Thread {id} failed to acquire the map ({ex.GetType().Name}: {ex.Message}), abort");
return;
}
}
logger.LogInformation($"Thread {id}: acquired the map");
while (!token.IsCancellationRequested)
{
try
{
var key = RandomProvider.Random.Next(0, entryCount).ToString();
var operation = RandomProvider.Random.Next(0, 100);
if (operation < 30)
{
await map.GetAsync(key).ConfigureAwait(false);
}
else if (operation < 60)
{
await map.PutAsync(key, RandomProvider.Random.Next().ToString()).ConfigureAwait(false);
}
else if (operation < 80)
{
await map.GetValuesAsync(Predicates.Value().IsBetween(0, 10)).ConfigureAwait(false);
}
else
{
await map.ExecuteAsync(new UpdateEntryProcessor(key), key).ConfigureAwait(false);
}
_reports[id] += 1;
}
catch (Exception ex)
{
logger.LogError($"Thead {id} caught {ex.GetType().Name}: {ex.Message}");
}
}
}
/// <summary>
/// Starts a new member and wait until it is added.
/// </summary>
/// <param name="rc">The remote controller.</param>
/// <param name="client">The Hazelcast client.</param>
/// <param name="cluster">The cluster.</param>
/// <param name="expectedPartitionOwnersCount">The expected number of partition owners.</param>
/// <returns>The new member.</returns>
public static async Task <Member> StartMemberWaitAddedAsync(this IRemoteControllerClient rc, IHazelcastClient client, Cluster cluster, int expectedPartitionOwnersCount)
{
var clientInternal = (HazelcastClient)client;
var added = new SemaphoreSlim(0);
var partitions = new SemaphoreSlim(0);
var subscriptionId = await clientInternal.SubscribeAsync(on => on
.MembersUpdated((sender, args) =>
{
if (args.AddedMembers.Count > 0)
{
added.Release();
}
})
.PartitionsUpdated((sender, args) =>
{
partitions.Release();
}))
.CfAwait();
var member = await rc.StartMemberAsync(cluster).CfAwait();
await added.WaitAsync(TimeSpan.FromSeconds(120)).CfAwait();
// trigger the partition table creation
var map = await client.GetMapAsync <object, object>("default").CfAwait();
_ = map.GetAsync(new object());
await partitions.WaitAsync(TimeSpan.FromSeconds(120)).CfAwait();
await clientInternal.UnsubscribeAsync(subscriptionId).CfAwait();
var partitioner = clientInternal.Cluster.Partitioner;
var partitionsCount = partitioner.Count;
var owners = new HashSet <Guid>();
for (var i = 0; i < partitionsCount; i++)
{
var owner = partitioner.GetPartitionOwner(i);
if (owner != default)
{
owners.Add(owner);
}
}
Assert.AreEqual(expectedPartitionOwnersCount, owners.Count);
return(member);
}
public async Task Init()
{
_client = await CreateAndStartClientAsync();
var client = _client as HazelcastClient;
Assert.That(client, Is.Not.Null);
SerializationService = client.SerializationService;
_map = await _client.GetMapAsync <object, object>("nc-" + TestUtils.RandomString());
var nearCache = GetNearCache(_map);
Assert.That(nearCache, Is.InstanceOf <NearCaching.NearCache>());
}
public async Task RunAsync()
{
_logger.LogDebug("debug");
_logger.LogInformation("debug");
_logger.LogWarning("debug");
await using var map = await _client.GetMapAsync <string, int>("test-map");
await map.SetAsync("key", 42);
var value = await map.GetAsync("key");
if (value != 42)
{
throw new Exception("Error!");
}
Console.WriteLine("It worked.");
// destroy the map
await _client.DestroyAsync(map);
}
public async Task NearCacheRecoversFromDistortions()
{
const string name = "nc-distortion";
const int size = 100000;
var stop = false;
// pre-populate the dictionary with (k,k) pairs for all keys
Logger.LogInformation("Populate...");
Assert.True((await PopulateMapFromServerAsync(name, size)).Success);
var dictionary = await _client.GetMapAsync <int, int>(name);
await using var _ = new AsyncDisposable(() => dictionary?.DestroyAsync() ?? default);
var cache = GetNearCache(dictionary);
Logger.LogInformation("Start tasks...");
var caching = Task.Run(async() =>
{
while (!stop)
{
// reads values, thus populating the cache
for (var i = 0; i < size && !stop; i++)
{
await dictionary.GetAsync(i);
}
}
Logger.LogInformation("Caching stopped.");
});
var distortingSequence = Task.Run(async() =>
{
while (!stop)
{
// randomly changes the sequence of a partition on the server
var response = await DistortRandomPartitionSequenceAsync(name);
Assert.True(response.Success, response.Message);
await Task.Delay(1000);
}
Logger.LogInformation("Distorting sequence stopped.");
});
var distortingUuid = Task.Run(async() =>
{
while (!stop)
{
// randomly changes the uuid of a partition on the server
var response = await DistortRandomPartitionUuidAsync();
Assert.That(response.Success, response.Message);
await Task.Delay(5000);
}
Logger.LogInformation("Distorting uuid stopped.");
});
var addingOnMember = Task.Run(async() =>
{
while (!stop)
{
// adds or update on the server
var key = RandomProvider.Random.Next(size);
var value = RandomProvider.Random.Next(int.MaxValue);
var response = await PutOnMemberAsync(key, value, name);
Assert.That(response.Success);
await Task.Delay(100);
}
Logger.LogInformation("Adding stopped.");
});
// let it run for 60 seconds
Logger.LogInformation("Run...");
await Task.Delay(60_000);
// stop
Logger.LogInformation("Stop tasks...");
stop = true;
await Task.WhenAll(caching, distortingSequence, distortingUuid, addingOnMember);
Logger.LogInformation("Assert...");
await AssertEx.SucceedsEventually(async() =>
{
// compare the values directly obtained from the server,
// with the values obtained from the client via the cache,
// everything should match
var memberValues = await GetAllValueFromMemberAsync(size, name);
for (var i = 0; i < size; i++)
{
var memberValue = memberValues[i] as int?;
var clientValue = await dictionary.GetAsync(i);
Assert.AreEqual(memberValue, clientValue, $"Bad value (i={i}, server={memberValue}, client={clientValue})");
}
}, 40_000, 500);
}
public async Task NearCacheClearedBySet(bool strict)
{
var dictionary = await _client.GetMapAsync <string, string>(_name);
await using var _ = new AsyncDisposable(() => dictionary?.DestroyAsync() ?? default);
var cache = GetNearCache(dictionary);
// run test
await RunTestInternal(dictionary);
// if not strict, take plenty of time to become consistent
if (!strict)
{
await Task.Delay(10_000);
}
var lastValue = _valuePut;
var valueStr = await dictionary.GetAsync(Key);
Assert.That(valueStr, Is.Not.Null);
var valueInt = int.Parse(valueStr);
// fail if not eventually consistent
string msg = null;
if (valueInt < lastValue)
{
msg = $"Err: getting {valueInt} instead of {lastValue}";
// flush Near Cache and re-fetch value
cache.Clear();
valueStr = await dictionary.GetAsync(Key);
Assert.That(valueStr, Is.Not.Null);
valueInt = int.Parse(valueStr);
// test again
if (valueInt < lastValue)
{
msg += $" (flushing the cache the did not fix the inconsistency, getting {valueInt}).";
}
else
{
msg += " (flushing the cache did fix the inconsistency).";
}
}
Logger.LogInformation($"Statistics: \n\tHits: {cache.Statistics.Hits}\n\tMisses: {cache.Statistics.Misses}\n\tStale: {cache.Statistics.StaleReads}\n\tEvict: {cache.Statistics.Evictions}");
// fail after stopping hazelcast instance
if (msg != null)
{
Logger.LogWarning(msg);
Assert.Fail(msg);
}
// fail if strict is required and assertion was violated
if (strict && _assertionViolationCount > 0)
{
msg = "Got " + _assertionViolationCount + " errors.";
Logger.LogWarning(msg);
Assert.Fail(msg);
}
}
public async Task NearCacheTracksRemoteChanges()
{
var dictionary = await _client.GetMapAsync <int, int>(_name);
await using var _ = new AsyncDisposable(dictionary.DestroyAsync);
var cache = GetNearCache(dictionary);
var tasks = new List <Task>();
var stop = false;
const int entryCount = 10;
const int writerTaskCount = 1;
const int readerTaskCount = 10;
// start tasks that add / update (thus invalidate) values on the server
for (var i = 0; i < writerTaskCount; i++)
{
tasks.Add(Task.Run(async() =>
{
while (!stop)
{
Assert.That((await PopulateMapWithRandomValueFromServerAsync(_name, entryCount)).Success);
await Task.Delay(100);
}
}));
}
// start tasks that read values
for (var i = 0; i < readerTaskCount; i++)
{
tasks.Add(Task.Run(async() =>
{
while (!stop)
{
for (var j = 0; j < entryCount && !stop; j++)
{
await dictionary.GetAsync(j);
}
await Task.Delay(100);
}
}));
}
// run for some time
await Task.Delay(8_000);
// stop tasks
stop = true;
await Task.WhenAll(tasks);
// assert that eventually all cache values will match what's on the member
await AssertEx.SucceedsEventually(async() =>
{
// 1. get all values, some should come from the cache
var cacheValues = new List <int>();
for (var i = 0; i < entryCount; i++)
{
cacheValues.Add(await dictionary.GetAsync(i));
}
// 2. get all values from server directly
var memberValues = await GetAllValueFromMemberAsync(entryCount, _name);
// after a while, all values in the cache will be same as server
for (var i = 0; i < entryCount; i++)
{
Assert.AreEqual(memberValues[i], cacheValues[i]);
}
}, 10000, 500);
}
public async Task TestNearCache() // CacheIsPopulatedByReads
{
var dictionary = await _client.GetMapAsync <object, object>("nc-" + TestUtils.RandomString());
await using var _ = new AsyncDisposable(dictionary.DestroyAsync);
var cache = GetNearCache(dictionary);
// add values to the dictionary
for (var i = 0; i < MaxSize; i++)
{
await dictionary.SetAsync("key" + i, "value" + i);
}
// near cache remains empty
Assert.That(cache.Count, Is.EqualTo(0));
// get values, this will populate near cache
var begin = DateTime.Now;
for (var i = 0; i < MaxSize; i++)
{
await dictionary.GetAsync("key" + i);
}
var firstRead = DateTime.Now - begin;
// get values again, this should read from near cache = be way faster
// TODO: we should have a way (instrumentation) to count server requests
begin = DateTime.Now;
for (var i = 0; i < MaxSize; i++)
{
await dictionary.GetAsync("key" + i);
}
var secondRead = DateTime.Now - begin;
// verify it's faster the second time
Assert.IsTrue(secondRead < firstRead);
// verify the cache contains all values
Assert.That(cache.Count, Is.EqualTo(MaxSize));
Assert.That(cache.Statistics.EntryCount, Is.EqualTo(MaxSize));
}