public async Task Cache_Dist_Test_InMemoryPerformance()
{
// Arrange
var cacheAccessor1 = new CacheAccessor(new CacheConfig {
IsAuditingEnabled = true
});
var cacheAccessor2 = new CacheAccessor(new CacheConfig {
IsAuditingEnabled = true
});
await cacheAccessor1.InitialiseIfNecessaryAsync();
await cacheAccessor2.InitialiseIfNecessaryAsync();
var bigList = new List <int>(Enumerable.Range(0, 100000));
await cacheAccessor1.SetAsync("biglist", bigList);
var v1 = await cacheAccessor1.GetWithMetaDataAsync <List <int> >("biglist"); // should return instantly from the originating cache accessor (it's in-memory)
var v2 = await cacheAccessor2.GetWithMetaDataAsync <List <int> >("biglist"); // should return with some delay from the new cache accessor (it's on the central cache)
var v3 = await cacheAccessor2.GetWithMetaDataAsync <List <int> >("biglist"); // should return instantly, having put the object into in-memory cache the last time it was called
Assert.IsNotNull(v1.Data);
Assert.IsTrue(v1.IsFromInMemoryCache, "Shoulda oughta a bin from the in-memory cache");
Assert.IsNotNull(v2.Data);
Assert.IsTrue(v2.IsFromCentralCacheServer || v2.IsFromInMemoryCache, "Shoulda oughta a bin from the central / memory cache");
Assert.IsNotNull(v3.Data);
Assert.IsTrue(v3.IsFromInMemoryCache, "Shoulda oughta a bin from the in-memory cache");
}
public async Task CacheData_WhenCacheIsUnavailable_ReturnDefaultValueAndCheckConnectionTimeout()
{
// Arrange
var tempCache = new CacheAccessor(new CacheConfig
{
ConnectionString = "invalid_connection_string"
}); // configured with an incorrect server and will return null on error
// Act
var testValue = await tempCache.GetAsync <string>("lovely_non_existent_item");
Thread.Sleep(50);
var status1 = tempCache.Status;
await tempCache.InitialiseIfNecessaryAsync();
var status2 = tempCache.Status;
// Assert
Assert.IsNull(testValue);
Assert.AreEqual(CacheAccessor.State.Connecting, status1);
Assert.AreEqual(CacheAccessor.State.NotConnected, status2);
Assert.IsTrue(tempCache.LastConnectionElapsedMilliseconds > 0 && tempCache.LastConnectionElapsedMilliseconds < 12000,
"The connection timeout should be more than 0 but less than 12 seconds. Actual timeouts are never inline with the stated max timeout, so for 10ms, the actual timeout is ~5s");
}
/// <summary>
/// Read and unprotect cache data
/// </summary>
/// <returns>Unprotected cache data</returns>
public byte[] ReadData(bool ignoreExceptions = true)
{
bool cacheFileExists = File.Exists(CacheFilePath);
_logger.LogInformation($"ReadData Cache file exists '{cacheFileExists}'");
byte[] data = null;
try
{
_logger.LogInformation($"Reading Data");
data = CacheAccessor.Read();
_logger.LogInformation($"Got '{data?.Length}' bytes from file storage");
}
catch (Exception e)
{
_logger.LogError($"An exception was encountered while reading data from the {nameof(MsalCacheStorage)} : {e}");
// It's unlikely that Clear will work, but try it anyway
Clear();
if (!ignoreExceptions)
{
throw;
}
}
return(data ?? new byte[0]);
}
public async Task Cache_NonDist_Test_That_Cache_Delete_Key_Works_OK()
{
// ARRANGE
var ca1 = new CacheAccessor(new CacheConfig {
IsAuditingEnabled = true, IsCentralCacheEnabled = false
});
await ca1.InitialiseIfNecessaryAsync();
var KEY = CreateGuid();
var VALUE = CreateGuid();
// ACT
await ca1.SetAsync(KEY, VALUE); // set the value
await ca1.DeleteAsync(KEY);
var evt1 = ca1.LastCachingEvent;
var v2 = await ca1.GetWithMetaDataAsync <string>(KEY);
// ASSERT
Assert.IsNull(v2.Data);
Assert.IsTrue(evt1 == eCacheEvent.KeyDeletedInMemory);
Assert.IsTrue(ca1.LastCachingEvent == eCacheEvent.KeyValueGotFromMemoryAttempt);
}
public async Task Cache_TestImmutableRefTypeObjectDoesNOTCauseAnException()
{
var ca1 = new CacheAccessor(new CacheConfig {
IsAuditingEnabled = true, IsCentralCacheEnabled = false
});
var ops = ca1;
await ca1.InitialiseIfNecessaryAsync();
await ops.SetAsync(Guid.NewGuid().ToString(), "this object is immutable", null);
}
public async Task CacheData_WhenCacheIsUnavailable_ReturnNull()
{
// Arrange
var tempCache = new CacheAccessor(new CacheConfig
{
ConnectionString = "invalid_connection_string"
}); // configured with an incorrect server and will return null on error
// Act
var testValue = await tempCache.GetAsync <string>("lovely_non_existent_item");
Assert.IsNull(testValue);
}
/// <summary>
/// Tries to write -> read -> clear a secret from the underlying persistence mechanism
/// </summary>
public void VerifyPersistence()
{
// do not use the _cacheAccessor for writing dummy data, as it might overwrite an actual token cache
var persitenceValidatationAccessor = CacheAccessor.CreateForPersistenceValidation();
try
{
_logger.LogInformation($"[Verify Persistence] Writing Data ");
persitenceValidatationAccessor.Write(Encoding.UTF8.GetBytes(PersistenceValidationDummyData));
_logger.LogInformation($"[Verify Persistence] Reading Data ");
var data = persitenceValidatationAccessor.Read();
if (data == null || data.Length == 0)
{
throw new MsalCachePersistenceException(
"Persistence check failed. Data was written but it could not be read. " +
"Possible cause: on Linux, LibSecret is installed but D-Bus isn't running because it cannot be started over SSH.");
}
string dataRead = Encoding.UTF8.GetString(data);
if (!string.Equals(PersistenceValidationDummyData, dataRead, StringComparison.Ordinal))
{
throw new MsalCachePersistenceException(
$"Persistence check failed. Data written {PersistenceValidationDummyData} is different from data read {dataRead}");
}
}
catch (InteropException e)
{
throw new MsalCachePersistenceException(
$"Persistence check failed. Reason: {e.Message}. OS error code {e.ErrorCode}.", e);
}
catch (Exception ex) when(!(ex is MsalCachePersistenceException))
{
throw new MsalCachePersistenceException("Persistence check failed. Inspect inner exception for details", ex);
}
finally
{
try
{
_logger.LogInformation($"[Verify Persistence] Clearing data");
persitenceValidatationAccessor.Clear();
}
catch (Exception e)
{
_logger.LogError($"[Verify Persistence] Could not clear the test data: " + e);
}
}
}
public async Task Cache_TestValueTypeObject_Roundtrips()
{
var cache = new CacheAccessor(new CacheConfig {
IsAuditingEnabled = true, IsCentralCacheEnabled = false
});
var ops = cache;
var key = Guid.NewGuid().ToString("N");
var value = 4658792;
await cache.InitialiseIfNecessaryAsync();
await ops.SetAsync(key, value, null);
var value2 = await cache.GetAsync <int>(key);
Assert.AreEqual(value, value2);
}
/// <summary>
/// Delete cache file
/// </summary>
public void Clear(bool ignoreExceptions = true)
{
try
{
_logger.LogInformation("Clearing the cache file");
CacheAccessor.Clear();
}
catch (Exception e)
{
_logger.LogError($"An exception was encountered while clearing data from {nameof(MsalCacheStorage)} : {e}");
if (!ignoreExceptions)
{
throw;
}
}
}
/// <summary>
/// Read and unprotect cache data
/// </summary>
/// <returns>Unprotected cache data</returns>
public byte[] ReadData()
{
byte[] data = null;
try
{
_logger.LogInformation($"Reading Data");
data = CacheAccessor.Read();
_logger.LogInformation($"Got '{data?.Length ?? 0}' bytes from file storage");
}
catch (Exception e)
{
_logger.LogError($"An exception was encountered while reading data from the {nameof(Storage)} : {e}");
throw;
}
return(data ?? new byte[0]);
}
/// <summary>
/// Protect and write cache data to file. It overrides existing data.
/// </summary>
/// <param name="data">Cache data</param>
public void WriteData(byte[] data)
{
if (data == null)
{
throw new ArgumentNullException(nameof(data));
}
try
{
_logger.LogInformation($"Got '{data?.Length}' bytes to write to storage");
CacheAccessor.Write(data);
}
catch (Exception e)
{
_logger.LogError($"An exception was encountered while writing data to {nameof(Storage)} : {e}");
throw;
}
}
public async Task Cache_TestCustomObjects_AreNotClones()
{
var cache = new CacheAccessor(new CacheConfig {
IsAuditingEnabled = true, IsCentralCacheEnabled = false
});
var ops = cache;
var key = Guid.NewGuid().ToString("N");
var value = new Employee(3456, "test");
await cache.InitialiseIfNecessaryAsync();
await ops.SetAsync(key, value, null);
var value2 = await cache.GetAsync <Employee>(key);
Assert.AreEqual(value.Id, value2.Id);
Assert.AreEqual(value.Name, value2.Name);
Assert.AreNotEqual(value, value2);
}
public async Task Cache_TestWithoutRedis()
{
var cache = new CacheAccessor(new CacheConfig {
IsAuditingEnabled = true, IsCentralCacheEnabled = false
});
var ops = cache;
var key = Guid.NewGuid().ToString("N");
var value = new Employee(3456, "test");
await cache.InitialiseIfNecessaryAsync();
await ops.SetAsync(key, value, null);
var value2 = await cache.GetAsync <Employee>(key);
Assert.AreEqual(cache.AuditLog[0].CachingEvent, eCacheEvent.KeySetInMemory);
Assert.AreEqual(cache.AuditLog[1].CachingEvent, eCacheEvent.KeyValueGotFromMemoryAttempt);
Assert.AreEqual(cache.AuditLog[2].CachingEvent, eCacheEvent.KeyValueGotFromMemory);
}
public async Task Cache_TestDictionaryObject_Roundtrips()
{
var cache = new CacheAccessor(new CacheConfig {
IsAuditingEnabled = true, IsCentralCacheEnabled = false
});
var ops = cache;
var key = Guid.NewGuid().ToString("N");
var value = (IDictionary <string, string>) new Dictionary <string, string> {
{ "k", "v" }
};
await cache.InitialiseIfNecessaryAsync();
await ops.SetAsync(key, value, null);
var value2 = await cache.GetAsync <Dictionary <string, string> >(key);
Assert.AreEqual(value, value2);
}
public async Task Cache_TestEnumerableObject_Roundtrips()
{
var cache = new CacheAccessor(new CacheConfig {
IsAuditingEnabled = true, IsCentralCacheEnabled = false
});
var ops = cache;
var key = Guid.NewGuid().ToString("N");
var value = (IEnumerable) new List <string> {
"this is a thing in the list"
};
await cache.InitialiseIfNecessaryAsync();
await ops.SetAsync(key, value, null);
var value2 = await cache.GetAsync <IEnumerable>(key);
Assert.AreEqual(value, value2);
}
public void CacheData_TestThatMultipleInitialisationsResultsInOne()
{
var tempCache = CacheAccessor.Create();
Parallel.For(0, 20, i =>
{
tempCache.InitialiseIfNecessaryAsync();
});
var connectingTask = tempCache.InitialiseIfNecessaryAsync();
var status1 = tempCache.Status;
connectingTask.Wait(); // wait for connection to complete
var status2 = tempCache.Status;
Assert.AreEqual(1, tempCache.ConnectionAttemptsCount, "Only 1 connection attempt should have been made in-spite of the bombardment of connection requests");
Assert.IsTrue(status1 == CacheAccessor.State.NotConnected || status1 == CacheAccessor.State.Connecting);
Assert.AreEqual(CacheAccessor.State.Connected, status2, "The cache should have connected by status2");
}
public async Task Cache_Dist_Test_That_Cache_Expiry_Works_OK()
{
// ARRANGE
var ca1 = new CacheAccessor(new CacheConfig {
IsAuditingEnabled = true
});
var ca2 = new CacheAccessor(new CacheConfig {
IsAuditingEnabled = true
});
await ca1.InitialiseIfNecessaryAsync();
await ca2.InitialiseIfNecessaryAsync();
var KEY = CreateGuid();
var VALUE = CreateGuid();
// ACT
await ca1.SetAsync(KEY, VALUE, TimeSpan.FromSeconds(5));
await ca2.WaitForEvent(eCacheEvent.KeySetInMemory);
var v1 = await ca2.GetWithMetaDataAsync <string>(KEY);
var v2 = await ca1.GetWithMetaDataAsync <string>(KEY);
await Task.Delay(5000);
var v3 = await ca2.GetWithMetaDataAsync <string>(KEY);
var v4 = await ca1.GetWithMetaDataAsync <string>(KEY);
Assert.IsNotNull(v1.Data);
Assert.IsNotNull(v2.Data);
Assert.IsTrue(v1.IsFromInMemoryCache);
Assert.IsTrue(v2.IsFromInMemoryCache);
Assert.IsNull(v3.Data);
Assert.IsNull(v4.Data);
Assert.IsTrue(v3.IsFromCentralCacheServer);
Assert.IsTrue(v4.IsFromCentralCacheServer);
}
public async Task Cache_Dist_Test_Caching_Byte_Arrays()
{
// ARRANGE
var ca1 = new CacheAccessor(new CacheConfig {
IsAuditingEnabled = true
});
var ca2 = new CacheAccessor(new CacheConfig {
IsAuditingEnabled = true
});
await ca1.InitialiseIfNecessaryAsync();
await ca2.InitialiseIfNecessaryAsync();
var KEY1 = CreateGuid();
var VALUE1 = new byte[] { 32, 34, 55 };
var KEY2 = CreateGuid();
var VALUE2 = new byte[] { 1, 11, 33 };
// ACT
await ca1.SetAsync(KEY1, VALUE1); // set the value
await ca2.WaitForEvent(eCacheEvent.KeySetInMemory);
await ca2.SetAsync(KEY2, VALUE2); // set the value
await ca1.WaitForEvent(eCacheEvent.KeySetInMemory);
var data1 = await ca2.GetWithMetaDataAsync <byte[]>(KEY1);
var data2 = await ca1.GetWithMetaDataAsync <byte[]>(KEY2);
Assert.AreEqual(VALUE1.Length, data1.Data.Length);
Assert.AreEqual(VALUE2.Length, data2.Data.Length);
Assert.AreEqual(VALUE1[0], data1.Data[0]);
Assert.AreEqual(VALUE1[1], data1.Data[1]);
Assert.AreEqual(VALUE1[2], data1.Data[2]);
Assert.AreEqual(VALUE2[0], data2.Data[0]);
Assert.AreEqual(VALUE2[1], data2.Data[1]);
Assert.AreEqual(VALUE2[2], data2.Data[2]);
}
public async Task Cache_Dist_Test_That_Cache_Delete_Key_Works_OK()
{
// ARRANGE
var ca1 = new CacheAccessor(new CacheConfig {
IsAuditingEnabled = true
});
var ca2 = new CacheAccessor(new CacheConfig {
IsAuditingEnabled = true
});
await ca1.InitialiseIfNecessaryAsync();
await ca2.InitialiseIfNecessaryAsync();
var KEY = CreateGuid();
var VALUE = CreateGuid();
// ACT
await ca1.SetAsync(KEY, VALUE); // set the value
// Wait for propagation to ca2
await ca2.WaitForEvent(eCacheEvent.KeySetInMemory);
var v1 = await ca2.GetWithMetaDataAsync <string>(KEY);
await ca2.DeleteAsync(KEY);
var propagationTimeInMS = await ca1.WaitForEvent(eCacheEvent.KeyDeletedInMemory);
var v2 = await ca1.GetWithMetaDataAsync <string>(KEY);
var v3 = await ca2.GetWithMetaDataAsync <string>(KEY);
// ASSERT
Assert.IsNotNull(v1.Data);
Assert.IsTrue(v1.IsFromInMemoryCache);
Assert.IsNull(v2.Data);
Assert.IsTrue(v2.IsFromCentralCacheServer);
Assert.IsNull(v3.Data);
Assert.IsTrue(v3.IsFromCentralCacheServer);
}
public void Cache_SetIsReallyAsync()
{
var cache = CacheAccessor.Create();
var key = Key();
var value = CreateGuid();
Task settingTask = cache.SetAsync(key, value);
List <bool> flags = new List <bool>();
while (true)
{
flags.Add(settingTask.IsCompleted);
if (settingTask.IsCompleted)
{
break;
}
Thread.Sleep(10);
}
Assert.IsTrue(flags.Where(x => x == false).Count() > 0, "There should be at least a few points where the setting task was running");
Assert.IsTrue(flags.Where(x => x).Count() > 0, "Some flags should indicate the task completed");
}
public async Task Cache_TestThatConnectIsAsync()
{
const long MAX_CONNECTION_TIME_MS = 60000;
// Arrange
var key = Key();
var value = CreateGuid();
// Put something in the cache, the close the connection
using (var cache = CacheAccessor.Create())
await cache.SetAsync(key, value);
// Re-create the cache
using (var cache = CacheAccessor.Create())
{
// Act
var cacheValue = await cache.GetAsync <string>(key);
Assert.IsNull(cacheValue);
var sw = Stopwatch.StartNew();
while (cache.Status == CacheAccessor.State.NotConnected || cache.Status == CacheAccessor.State.Connecting)
{
await Task.Delay(500);
Assert.IsFalse(sw.ElapsedMilliseconds > MAX_CONNECTION_TIME_MS, "Connecting took too long ( > {0})", sw.ElapsedMilliseconds);
}
sw.Stop();
Assert.IsFalse(sw.ElapsedMilliseconds > MAX_CONNECTION_TIME_MS, "Connecting took too long ( > {0})", sw.ElapsedMilliseconds);
cacheValue = await cache.GetAsync <string>(key);
Assert.AreEqual(value, cacheValue);
await cache.DeleteAsync(key);
}
}
protected IDirectoryManager FactoryManager(ConfigSettingsDto settings = null, IDbContextFactory <LogContext> logCtxFactory = null, IDbContextFactory <DirectoryContext> dirCtxFactory = null, IDbContextFactory <ExportContext> xprtCtxFactory = null)
{
if (settings == null)
{
settings = new ConfigSettingsDto
{
StateCacheTtlSeconds = 1200,
TranslationCacheTtlSeconds = 1200
}
}
;
if (logCtxFactory == null)
{
logCtxFactory = new SqliteInMemoryContextFactory <LogContext>();
}
if (dirCtxFactory == null)
{
dirCtxFactory = new SqliteInMemoryContextFactory <DirectoryContext>();
}
if (xprtCtxFactory == null)
{
xprtCtxFactory = new SqliteInMemoryContextFactory <ExportContext>();
}
var cacheAccessor = new CacheAccessor(new MemoryCache(), settings);
var dirAccessor = new DirectoryAccessor(dirCtxFactory);
var logAccessor = new LogAccessor(logCtxFactory);
var exportAccessor = new ExportAccessor(xprtCtxFactory);
var dirEngine = new DirectoryEngine(dirAccessor, cacheAccessor, exportAccessor);
return(new DirectoryManager(dirEngine, logAccessor));
}
}
public async Task Cache_Dist_TestKeyValuePairPropagation()
{
// Arrange
var KEY = Key();
var VALUE = CreateGuid();
var cacheAccessor1 = new CacheAccessor(new CacheConfig {
IsAuditingEnabled = true
});
var cacheAccessor2 = new CacheAccessor(new CacheConfig {
IsAuditingEnabled = true
});
await cacheAccessor1.InitialiseIfNecessaryAsync();
await cacheAccessor2.InitialiseIfNecessaryAsync();
cacheAccessor1.AuditLog.Clear();
cacheAccessor2.AuditLog.Clear();
// Act
var v1 = await cacheAccessor1.GetWithMetaDataAsync <string>(KEY);
await cacheAccessor1.SetAsync(KEY, VALUE, TimeSpan.FromMinutes(2));
await Task.Delay(300);
var logOfCA2 = cacheAccessor2.AuditLog.ToArray();
var v2 = await cacheAccessor2.GetWithMetaDataAsync <string>(KEY);
var v3 = await cacheAccessor2.GetWithMetaDataAsync <string>(KEY);
var cacheAccessor3 = new CacheAccessor(new CacheConfig());
await cacheAccessor3.InitialiseIfNecessaryAsync();
var v6 = await cacheAccessor3.GetWithMetaDataAsync <string>(KEY);
var v7 = await cacheAccessor3.GetWithMetaDataAsync <string>(KEY);
await cacheAccessor2.DeleteAsync(KEY);
var v4_1 = await cacheAccessor1.GetWithMetaDataAsync <string>(KEY);
var v5_1 = await cacheAccessor2.GetWithMetaDataAsync <string>(KEY);
await Task.Delay(300);
var v4 = await cacheAccessor1.GetWithMetaDataAsync <string>(KEY);
var v5 = await cacheAccessor2.GetWithMetaDataAsync <string>(KEY);
Assert.IsTrue(logOfCA2.First().CachingEvent == eCacheEvent.KeySetInMemory, "The next audit event should have been to receive the key update message");
Assert.IsTrue(v1.IsFromCentralCacheServer, "Trying to get something that isn't set, should _AT_LEAST_ try the central cache");
Assert.IsTrue(v2.IsFromInMemoryCache, "Getting an item after setting it should mean that the in-memory cache has had time to catch up");
Assert.IsTrue(v3.IsFromInMemoryCache, "The value should have propagated to cache accessor");
Assert.IsNull(v4.Data, "The value should be null");
Assert.IsNull(v5.Data, "The value should be null");
Assert.IsTrue(v4.IsFromCentralCacheServer, "If a value is null, it should have tried the central cache server");
Assert.IsTrue(v5.IsFromCentralCacheServer, "If a value is null, it should have tried the central cache server");
Assert.IsNotNull(v6.Data, "The data shouldn't be null");
Assert.IsTrue(v6.IsFromCentralCacheServer, "The value should have been retrieved from the central server");
Assert.IsNotNull(v7.Data, "The data shouldn't be null");
Assert.AreEqual(VALUE, v7.Data);
Assert.IsTrue(v7.IsFromInMemoryCache, "The value should have been retrieved from the memory cache");
// This test is too subjective for the build server
// Assert.IsTrue(v7.ElapsedMilliseconds < v6.ElapsedMilliseconds, "It should have been quicker to get the value the second time");
}
public async Task Cache_Dist_Test_LargeVolumeOfCacheAccessors()
{
// Arrange
var inits = new List <Task>();
var caches = new List <CacheAccessor>();
const int MAX_CACHES = 20;
// Create MAX_CACHES cache accessors and therefore MAX_CACHES connections to redis
for (int i = 0; i < MAX_CACHES; i++)
{
var c = new CacheAccessor(new CacheConfig {
IsAuditingEnabled = true
});
inits.Add(c.InitialiseIfNecessaryAsync());
caches.Add(c);
}
await Task.WhenAll(inits.ToArray()); // wait for initialisation
var cacheOne = caches.First();
var KEY = Key();
var VALUE = new List <int>(Enumerable.Range(0, 1000));
// Clear audit logs of all caches
caches.ForEach(x => x.AuditLog.Clear());
// ACT
await cacheOne.SetAsync(KEY, VALUE);
var valueFromOriginator = await cacheOne.GetWithMetaDataAsync <List <int> >(KEY);
// Wait until the value has propagated
var propagationTasks = new List <Task>();
foreach (var cacher in caches)
{
if (cacher == cacheOne)
{
continue; // ignore the originating cacher
}
propagationTasks.Add(cacher.WaitForEvent(eCacheEvent.KeySetInMemory, 60000));
}
await Task.WhenAll(propagationTasks.ToArray());
var values = new List <CacheResponseDto <List <int> > >();
for (int i = 0; i < MAX_CACHES; i++)
{
var currentCache = caches[i];
values.Add(await currentCache.GetWithMetaDataAsync <List <int> >(KEY));
}
// ASSERT
Assert.AreEqual(VALUE, valueFromOriginator.Data);
Assert.IsTrue(valueFromOriginator.IsFromInMemoryCache);
foreach (var item in values)
{
Assert.AreEqual(VALUE, item.Data);
Assert.IsTrue(item.IsFromInMemoryCache);
}
for (int i = 1; i < MAX_CACHES; i++)
{
Assert.IsTrue(caches[i].AuditLog.First().CachingEvent == eCacheEvent.KeySetInMemory);
}
}
public void Setup()
{
// Create cache that is actually connected
_cacheThatsConnected = CacheAccessor.Create();
_cacheThatsConnected.InitialiseIfNecessaryAsync().Wait();
}