static void Main(string[] args) { var rndSrc = new Random(2); var testIdSrc = 0; var uniqueCacheId = 1; var ttl = TimeSpan.FromSeconds(10); // 20 min equiv var regen = TimeSpan.FromSeconds(2.5); // 1 min equiv var regenDuration = TimeSpan.FromSeconds(1.25); // duration of call to generate value to store in cache var regenErrorPercentage = 0; var farmClockTollerance = 0.1; var p1Duration = TimeSpan.FromSeconds(10); var p1Frequency = TimeSpan.FromSeconds(0.5); var launchTwoInstances = false; var doRegTest = true; var doPerfTest = true; var writePerformanceCsv = false; var writeTraceOutputFile = false; var writeHtmlOutputFile = false; var outputFolder = "c:\\temp\\"; var perfTestCount = 1000000; var perfTestMeasure = 100000; var keyspace = $"keyspace{Guid.NewGuid():N}"; if (launchTwoInstances && args.Length == 0 && !Debugger.IsAttached) { Process.Start(Assembly.GetEntryAssembly().Location, $"nospawn 1 {keyspace}"); //Task.Delay(2000).Wait(); Process.Start(Assembly.GetEntryAssembly().Location, $"nospawn 2 {keyspace}"); return; } var id = 0; if (args.Length >= 3) { int.TryParse(args[1], out id); keyspace = args[2]; } var synchedConsole = new SynchedColouredConsoleTraceWriter( traceFileName: writeTraceOutputFile ? $"{outputFolder}RegenerativeCacheManagerDemo_{keyspace}_{id}.txt" : null, htmlFileName: writeHtmlOutputFile ? $"{outputFolder}RegenerativeCacheManagerDemo_{keyspace}_{id}.html" : null, performanceFileName: writePerformanceCsv ? $"{outputFolder}RegenerativeCacheManagerDemo_{keyspace}_{id}.csv" : null ); using (var ct1R = new BasicRedisWrapper("localhost")) using (var cacheTest1 = new RegenerativeCacheManager("keyspace3", ct1R.Cache, ct1R.Lock, ct1R.Bus)) using (var ct2R = new BasicRedisWrapper("localhost")) using (var cacheTest2 = new RegenerativeCacheManager("Keyspace3", ct2R.Cache, ct2R.Lock, ct2R.Bus)) { new List <IDisposable> { cacheTest1, ct1R, cacheTest2, ct2R }.ForEach(d => d.Dispose()); } synchedConsole.WriteLine("RegenerativeCacheManager dispose works.", ConsoleColor.Black, ConsoleColor.Green); var redis = new BasicRedisWrapper("localhost", true); var cache = new RegenerativeCacheManager(keyspace, redis.Cache, redis.Lock, redis.Bus, synchedConsole) { // low tollerences in this test due to extremely low cache item re-generation and expiry times, normally 30s to minutes CacheExpiryToleranceSeconds = 1, TriggerDelaySeconds = 1, MinimumForwardSchedulingSeconds = 1, FarmClockToleranceSeconds = farmClockTollerance, }; var generateFunc = new Func <string>(() => { Task.Delay(regenDuration).Wait(); if (rndSrc.Next(0, 100) < regenErrorPercentage) { throw new ApplicationException("Synthetic Error"); } return($" >>> CacheItemVal_{DateTime.Now:ss.fff}_{Interlocked.Increment(ref uniqueCacheId)} <<< "); }); if (doRegTest) { synchedConsole.WriteLine($"Inactive TTL: {ttl.TotalSeconds}s, Regeneration every: {regen.TotalSeconds}s, Regeneration performance: {regenDuration.TotalSeconds}s"); synchedConsole.WriteLine($"------- running get per second (in background) for {p1Duration.TotalSeconds} seconds"); synchedConsole.WriteLine("================================================="); var start = DateTime.UtcNow; while (DateTime.UtcNow.Subtract(start).TotalSeconds < p1Duration.TotalSeconds) { var testId = Interlocked.Increment(ref testIdSrc); Task.Run(() => synchedConsole.WriteLine($"* PART 1 * Cache Value: {MonitorWork(() => cache.GetOrAdd("test1", generateFunc, ttl, regen), synchedConsole, testId)}")); Task.Delay(p1Frequency).Wait(); } synchedConsole.WriteLine("================================================="); synchedConsole.DebugWait(6); var testX1 = Interlocked.Increment(ref testIdSrc); synchedConsole.WriteLine($"* PART 2 * Cache Value: {MonitorWork(() => cache.GetOrAdd("test1", generateFunc, ttl, regen), synchedConsole, testX1)}"); synchedConsole.WriteLine("================================================="); synchedConsole.DebugWait(15); var testX2 = Interlocked.Increment(ref testIdSrc); Task.Run(() => synchedConsole.WriteLine($"* PART 3 * Cache Value: {MonitorWork(() => cache.GetOrAdd("test1", generateFunc, ttl, regen), synchedConsole, testX2)}")); var testX3 = Interlocked.Increment(ref testIdSrc); Task.Run(() => synchedConsole.WriteLine($"* PART 3 * Cache Value: {MonitorWork(() => cache.GetOrAdd("test1", generateFunc, ttl, regen), synchedConsole, testX3)}")); var testX4 = Interlocked.Increment(ref testIdSrc); synchedConsole.WriteLine($"* PART 3 * Cache Value: {MonitorWork(() => cache.GetOrAdd("test1", generateFunc, ttl, regen), synchedConsole, testX4)}"); var testX5 = Interlocked.Increment(ref testIdSrc); synchedConsole.WriteLine($"* PART 3 * Cache Value: {MonitorWork(() => cache.GetOrAdd("test1", generateFunc, ttl, regen), synchedConsole, testX5)}"); synchedConsole.DebugWait(2); synchedConsole.DebugWait(20); ShowStats(synchedConsole); try { Task.WaitAll(MonitoredWorkBag.Select(t => t.Item1).ToArray()); } catch (Exception ex) { synchedConsole.WriteLine($"First error: {(ex as AggregateException)?.InnerExceptions.First().ToString() ?? ex.ToString()}", ConsoleColor.White, ConsoleColor.Red); } while (MonitoredWorkBag.TryTake(out Tuple <Task, TimeSpan> tr)) { tr.Item1.Dispose(); } } if (doRegTest && doPerfTest) { synchedConsole.OpenNewOutputFile(); } if (doPerfTest) { cache.Dispose(); _perTestMeasureEveryN = Math.Max(1, perfTestCount / perfTestMeasure); cache = new RegenerativeCacheManager(keyspace, redis.Cache, redis.Lock, redis.Bus, synchedConsole) { // low tollerences in this test due to extremely low cache item re-generation and expiry times, normally 30s to minutes TriggerDelaySeconds = 1, MinimumForwardSchedulingSeconds = 1, FarmClockToleranceSeconds = farmClockTollerance, CacheExpiryToleranceSeconds = 5, }; synchedConsole.WriteLine($"Running performance test of {perfTestCount:#,##0} cache calls to GetOrAdd in parallel.", ConsoleColor.White, ConsoleColor.DarkBlue); testIdSrc = 1; while (MonitoredWorkBag.TryTake(out Tuple <Task, TimeSpan> tr)) { ; } synchedConsole.ShowFullOutputToConsole = false; // warm up cache.GetOrAdd("test1", generateFunc, ttl, regen); var swWait = Stopwatch.StartNew(); var dop = 10; Parallel.ForEach(Enumerable.Range(0, dop), new ParallelOptions { MaxDegreeOfParallelism = dop }, (i) => { for (int j = 0; j < perfTestCount / dop; j++) { var testId = Interlocked.Increment(ref testIdSrc); if (j % 10 == 0) { Thread.Sleep(1); } synchedConsole.WriteLine($"* PART 4 * Cache Value: {MonitorWork(() => cache.GetOrAdd("test1", generateFunc, ttl, regen), synchedConsole, testId)}"); } }); swWait.Stop(); synchedConsole.WriteLine($"\r\n\r\n\t\t\t{perfTestCount:#,##0} cache GetOrAdd calls completed with {dop} degrees of parallelism." + $"\r\n\r\n\t\t\t Duration: {swWait.Elapsed.TotalMilliseconds * 1000:#,##0.0}us." + $"\r\n\t\t\t Requests/s: {perfTestCount / swWait.Elapsed.TotalSeconds:#,##0.0} ({perfTestCount / swWait.Elapsed.TotalHours:#,##0}/h)" + $"\r\n\r\n" , ConsoleColor.White, overrideShowOutput: true); ShowStats(synchedConsole); while (MonitoredWorkBag.TryTake(out Tuple <Task, TimeSpan> tr)) { tr.Item1.Dispose(); } GC.Collect(); } synchedConsole.CloseAndStopAllWriting(); if (launchTwoInstances) { synchedConsole.WriteLine("Press enter to exit.", overrideShowOutput: true); Console.ReadLine(); } }
private static void MultiNodeGetsInternal(string redisConnection, bool useMultipleRedisConnections, ITestOutputHelper _output, string writeTraceFile = null) { var testRunKeyspace = $"testKs{DateTime.Now:mmssfffffff}s{Interlocked.Increment(ref _seq)}"; if (redisConnection.ToLowerInvariant() == "mock") { redisConnection = $"mock-{Guid.NewGuid():N}"; } var tw = new DualTraceWriter(); try { using (var node1Ext = new RedisInterceptOrMock(redisConnection, useMultipleRedisConnections)) using (var node1Cache = new RegenerativeCacheManager(testRunKeyspace, node1Ext.Cache, node1Ext.Lock, node1Ext.Bus, tw.T1) { CacheExpiryToleranceSeconds = 1.5, MinimumForwardSchedulingSeconds = 1, TriggerDelaySeconds = 1, FarmClockToleranceSeconds = 0.1, }) using (var node2Ext = new RedisInterceptOrMock(redisConnection, useMultipleRedisConnections)) using (var node2Cache = new RegenerativeCacheManager(testRunKeyspace, node2Ext.Cache, node2Ext.Lock, node2Ext.Bus, tw.T2) { CacheExpiryToleranceSeconds = 1.5, MinimumForwardSchedulingSeconds = 1, TriggerDelaySeconds = 1, FarmClockToleranceSeconds = 0.1, }) { var inactiveRetention = TimeSpan.FromSeconds(6); var regenerationInterval = TimeSpan.FromSeconds(2); var node1Result1 = node1Cache.GetOrAdd("test1", () => MockGenDelay($"t1n1_{Guid.NewGuid():N}"), inactiveRetention, regenerationInterval); var node1Result2 = node1Cache.GetOrAdd("test1", () => MockGenDelay($"t1n1_{Guid.NewGuid():N}"), inactiveRetention, regenerationInterval); Assert.StartsWith("t1n1_", node1Result1); Assert.Equal(node1Result1, node1Result2); Assert.Single(node1Ext.CacheGets); Assert.Single(node1Ext.CacheGetStringStarts); Assert.Single(node1Ext.CacheSets); Assert.Single(node1Ext.Subscribes); Assert.Single(node1Ext.Publishes); Assert.Single(node1Ext.LockAttempts); Assert.Single(node1Ext.LockAttempts.Where(l => l.Value)); Assert.Empty(node2Ext.CacheGets); Assert.Empty(node2Ext.CacheGetStringStarts); Assert.Empty(node2Ext.CacheSets); Assert.Single(node2Ext.Subscribes); Assert.Empty(node2Ext.Publishes); Assert.Empty(node2Ext.LockAttempts); Thread.Sleep(250); Assert.Single(node1Ext.ReceivedMessages); Assert.Single(node2Ext.ReceivedMessages); var node2Result1 = node2Cache.GetOrAdd("test1", () => MockGenDelay($"t1n2_{Guid.NewGuid():N}"), inactiveRetention, regenerationInterval); var node2Result2 = node2Cache.GetOrAdd("test1", () => MockGenDelay($"t1n2_{Guid.NewGuid():N}"), inactiveRetention, regenerationInterval); Assert.Single(node1Ext.CacheSets); Assert.Single(node1Ext.Subscribes); Assert.Single(node1Ext.Publishes); Assert.Single(node1Ext.ReceivedMessages); Assert.Single(node1Ext.LockAttempts); Assert.Single(node1Ext.LockAttempts.Where(l => l.Value)); Assert.Single(node2Ext.CacheGets); Assert.Empty(node2Ext.CacheSets); Assert.Single(node2Ext.Subscribes); Assert.Empty(node2Ext.Publishes); Assert.Single(node2Ext.ReceivedMessages); Assert.Empty(node2Ext.LockAttempts); // confirm node2 gets result generated on node1 from external cache Assert.StartsWith("t1n1_", node2Result1); Assert.Equal(node2Result1, node2Result2); Thread.Sleep(4000); // confirm cache has been regenerated in the background (on both nodes) // one set from initial and one from regeneration (across farm) Assert.InRange(node1Ext.CacheSets.Count + node2Ext.CacheSets.Count, 2, 4); Assert.Single(node1Ext.Subscribes); var totalPublished = node1Ext.Publishes.Count + node2Ext.Publishes.Count; Assert.InRange(totalPublished, 2, 4); Assert.Equal(totalPublished, node1Ext.ReceivedMessages.Count); Assert.Equal(totalPublished, node2Ext.ReceivedMessages.Count); // two to four total lock attempt (depending on thread race conditions) Assert.InRange(node1Ext.LockAttempts.Count + node2Ext.LockAttempts.Count, 2, 6); // two successful locks across the farm (initial generation + 1 regeneration) Assert.InRange(node1Ext.LockAttempts.Count(l => l.Value) + node2Ext.LockAttempts.Count(l => l.Value), 2, 4); var locksAcquired = node1Ext.LockAttempts.Count(l => l.Value) + node2Ext.LockAttempts.Count(l => l.Value); // confirm first hit to node2 returns val from node1 Assert.Single(node2Ext.Subscribes); // initial generation + 1 re-generation per node Assert.InRange(node1Ext.ReceivedMessages.Count, locksAcquired - 1, locksAcquired); Assert.InRange(node2Ext.ReceivedMessages.Count, locksAcquired - 1, locksAcquired); var node1Result3 = node1Cache.GetOrAdd("test1", () => MockGenDelay($"t2n1_{Guid.NewGuid():N}"), inactiveRetention, regenerationInterval); var node1Result4 = node1Cache.GetOrAdd("test1", () => MockGenDelay($"t2n1_{Guid.NewGuid():N}"), inactiveRetention, regenerationInterval); // result still starts with t1n1_ or t1n2 Assert.NotEqual(node1Result1, node1Result3); Assert.StartsWith("t1", node1Result3); Assert.Equal(node1Result3, node1Result4); Thread.Sleep(10000); var node1Result5 = node1Cache.GetOrAdd("test1", () => MockGenDelay($"t3n1_{Guid.NewGuid():N}"), inactiveRetention, regenerationInterval); var node1Result6 = node1Cache.GetOrAdd("test1", () => MockGenDelay($"t3n1_{Guid.NewGuid():N}"), inactiveRetention, regenerationInterval); // result still start with t3n1 Assert.StartsWith("t3n1_", node1Result5); Assert.Equal(node1Result5, node1Result6); Thread.Sleep(10000); var node2Result3 = node1Cache.GetOrAdd("test1", () => MockGenDelay($"t4n2_{Guid.NewGuid():N}"), inactiveRetention, regenerationInterval); var node2Result4 = node1Cache.GetOrAdd("test1", () => MockGenDelay($"t4n2_{Guid.NewGuid():N}"), inactiveRetention, regenerationInterval); var node1Result7 = node1Cache.GetOrAdd("test1", () => MockGenDelay($"t4n1_{Guid.NewGuid():N}"), inactiveRetention, regenerationInterval); var node1Result8 = node1Cache.GetOrAdd("test1", () => MockGenDelay($"t4n1_{Guid.NewGuid():N}"), inactiveRetention, regenerationInterval); // check results flow from node2 back to node1 as well Assert.StartsWith("t4n2_", node2Result3); Assert.Equal(node2Result3, node2Result4); Assert.Equal(node2Result3, node1Result7); Assert.Equal(node2Result3, node1Result8); tw.Stop(); tw.Clear(); } } finally { foreach (var l in tw.GetOutput()) { _output.WriteLine(l); } } }
private static void NodesCompeteInternal(string redisConnection, bool useMultipleRedisConnections, ITestOutputHelper _output, string writeTraceFile = null) { var testRunKeyspace = $"testKs{DateTime.Now:mmssfffffff}s{Interlocked.Increment(ref _seq)}"; if (redisConnection.ToLowerInvariant() == "mock") { redisConnection = $"mock-{Guid.NewGuid():N}"; } var tw = new DualTraceWriter(); try { using (var node1Ext = new RedisInterceptOrMock(redisConnection, useMultipleRedisConnections)) using (var node1Cache = new RegenerativeCacheManager(testRunKeyspace, node1Ext.Cache, node1Ext.Lock, node1Ext.Bus, tw.T1) { CacheExpiryToleranceSeconds = 1.5, MinimumForwardSchedulingSeconds = 1, TriggerDelaySeconds = 1, FarmClockToleranceSeconds = 0.1, }) using (var node2Ext = new RedisInterceptOrMock(redisConnection, useMultipleRedisConnections)) using (var node2Cache = new RegenerativeCacheManager(testRunKeyspace, node2Ext.Cache, node2Ext.Lock, node2Ext.Bus, tw.T2) { CacheExpiryToleranceSeconds = 1.5, MinimumForwardSchedulingSeconds = 1, TriggerDelaySeconds = 1, FarmClockToleranceSeconds = 0.1, }) { var inactiveRetention = TimeSpan.FromSeconds(9); var regenerationInterval = TimeSpan.FromSeconds(3); bool seenN1 = false, seenN2 = false; var start = DateTime.Now; var end = start.AddSeconds(120); var rnd = new Random(); bool nodeOrder = false; string node1Result1 = "", node2Result1 = ""; var getFrom1 = new Action(() => node1Result1 = node1Cache.GetOrAdd("test1", () => MockGenDelay($"t1n1_{Guid.NewGuid():N}"), inactiveRetention, regenerationInterval)); var getFrom2 = new Action(() => node2Result1 = node2Cache.GetOrAdd("test1", () => MockGenDelay($"t1n2_{Guid.NewGuid():N}"), inactiveRetention, regenerationInterval)); var results = new List <Tuple <string, string> >(); do { // ReSharper disable once AssignmentInConditionalExpression if (nodeOrder = !nodeOrder) { getFrom1(); getFrom2(); } else { getFrom2(); getFrom1(); } results.Add(new Tuple <string, string>(node2Result1, node2Result1)); if (node2Result1.StartsWith("t1n1") || node1Result1.StartsWith("t1n1")) { seenN1 = true; } if (node2Result1.StartsWith("t1n2") || node1Result1.StartsWith("t1n2")) { seenN2 = true; } // randomise the delay to randomise the node order switches Task.Delay(rnd.Next(50, 100)).Wait(); } while (DateTime.Now < end && (!seenN1 || !seenN2)); var countOfEqual = results.Count(r => r.Item1 == r.Item2) * 1.0; var probabilityMsg = $"{end.Subtract(start).TotalSeconds:0} seconds means " + $"{(int)(end.Subtract(start).TotalSeconds / regenerationInterval.TotalSeconds)} chances to compete, " + $"chances of not seeing results from both nodes is approximately " + $"2 in (2^{(int)(end.Subtract(start).TotalSeconds / regenerationInterval.TotalSeconds)}), " + $"i.e. 1 in {Math.Pow(2, (int)(end.Subtract(start).TotalSeconds / regenerationInterval.TotalSeconds)) / 2:#,##0}."; // _output.WriteLine($"90% of results should be identical, {(countOfEqual / results.Count) * 100:0}% were ({countOfEqual} / {results.Count}). {probabilityMsg}"); Assert.True(results.Count > 19, $"Shouldn't see any cache regeneration / node competition for 2 to 3 seconds. Saw different results in only {DateTime.Now.Subtract(start).TotalMilliseconds*1000:#,##0.0}us."); Assert.True(countOfEqual / results.Count > 0.9, $"90% of results should be identical, only {(countOfEqual / results.Count)*100:0}% were ({countOfEqual} / {results.Count})."); var errmsg = $"Did not see generation on {{0}} - {probabilityMsg}"; Assert.True(seenN1, string.Format(errmsg, "node1")); Assert.True(seenN2, string.Format(errmsg, "node2")); } tw.Stop(); tw.Clear(); } finally { foreach (var l in tw.GetOutput()) { _output.WriteLine(l); } } }
private static void SingleNodeGetsInternal(string redisConnection, bool useMultipleRedisConnections, ITestOutputHelper _output, string writeTraceFile = null) { var testRunKeyspace = $"testKs{DateTime.Now:mmssfffffff}s{Interlocked.Increment(ref _seq)}"; if (redisConnection.ToLowerInvariant() == "mock") { redisConnection = $"mock-{Guid.NewGuid():N}"; } var tw = new DualTraceWriter(); try { using (var ext = new RedisInterceptOrMock(redisConnection, useMultipleRedisConnections)) using (var cache = new RegenerativeCacheManager(testRunKeyspace, ext.Cache, ext.Lock, ext.Bus, tw.T1) { CacheExpiryToleranceSeconds = 1.5, MinimumForwardSchedulingSeconds = 1, TriggerDelaySeconds = 1, FarmClockToleranceSeconds = 0.1, }) { var inactiveRetention = TimeSpan.FromSeconds(6); var regenerationInterval = TimeSpan.FromSeconds(2); var result1 = cache.GetOrAdd("test1", () => $"t1_{Guid.NewGuid():N}", inactiveRetention, regenerationInterval); var result2 = cache.GetOrAdd("test1", () => $"t1_{Guid.NewGuid():N}", inactiveRetention, regenerationInterval); Assert.StartsWith("t1_", result1); Assert.Equal(result1, result2); Assert.Single(ext.CacheGets); Assert.Single(ext.CacheGetStringStarts); Assert.Single(ext.CacheSets); Assert.Single(ext.Subscribes); Assert.Single(ext.Publishes); Assert.Single(ext.LockAttempts); Assert.Single(ext.LockAttempts.Where(l => l.Value)); Thread.Sleep(350); Assert.Single(ext.ReceivedMessages); // Confirm external cache keys are as expected Assert.NotNull(ext.StringGetWithExpiry($"{nameof(MemoryFrontedExternalCache)}:{testRunKeyspace}:Item:test1", out TimeSpan exp)); var first10 = ext.GetStringStart($"{nameof(MemoryFrontedExternalCache)}:{testRunKeyspace}:Item:test1", 10); var first100 = ext.GetStringStart($"{nameof(MemoryFrontedExternalCache)}:{testRunKeyspace}:Item:test1", 100); Assert.NotNull(first10); Assert.NotNull(first100); Assert.StartsWith(first10, first100); Thread.Sleep(5000); // confirm cache has been regenerated in the background Assert.InRange(ext.CacheSets.Count, 2, 4); Assert.Single(ext.Subscribes); Assert.InRange(ext.Publishes.Count, 2, 4); Assert.InRange(ext.ReceivedMessages.Count, 2, 4); Assert.InRange(ext.LockAttempts.Count, 2, 4); Assert.InRange(ext.LockAttempts.Count(l => l.Value), 2, 4); var result3 = cache.GetOrAdd("test1", () => $"t2_{Guid.NewGuid():N}", inactiveRetention, regenerationInterval); var result4 = cache.GetOrAdd("test1", () => $"t2_{Guid.NewGuid():N}", inactiveRetention, regenerationInterval); // result still starts with t1_ as we have not passed the 6s mark // but has been regenerated so is not equal to previous value Assert.NotEqual(result1, result3); Assert.StartsWith("t1_", result3); Assert.Equal(result3, result4); Thread.Sleep(10000); // 6s (regens) + 2 (validity) + 1.5 (tolerance) var result5 = cache.GetOrAdd("test1", () => $"t3_{Guid.NewGuid():N}", inactiveRetention, regenerationInterval); var result6 = cache.GetOrAdd("test1", () => $"t3_{Guid.NewGuid():N}", inactiveRetention, regenerationInterval); // result still starts with t1_ as we have not passed the 3s mark Assert.StartsWith("t3_", result5); Assert.Equal(result5, result6); tw.Stop(); tw.Clear(); } } finally { foreach (var l in tw.GetOutput()) { _output.WriteLine(l); } } }