public void BasicCreateAndClean()
{
Config config = ObjectTestHelper.GetConfigWithCaching();
ObjectMetadataStore metadata = new ObjectMetadataStore(config);
ObjectVersionStore versions = new ObjectVersionStore(config, metadata);
ObjectIndexerCache cache = new ObjectIndexerCache(metadata, versions);
string objectName = "TestObj";
try
{
// Load up the cache - first we will test that it is not cleaned when it shouldn't be
for (int i = 0; i < 20; i++) // generate 20 queries and 80 object Ids
{
cache.Set(
new int[] { 1, 2, 3, 4 },
new object[] { objectName, "Idx" + i + "=2" });
}
// Verify the totals
Assert.AreEqual(20, cache.EnumerateCache().Count());
CacheTotals totals = cache.Totals;
Assert.AreEqual(20, totals.TotalQueries);
Assert.AreEqual(80, totals.TotalValues);
Assert.IsNotNull(cache.Get(new object[] { objectName, "Idx1=2" }));
using (var cleaner = new HardPruneCacheCleaner(cache,
50,
200,
2,
500)) // run every half second
{
// verify properties
Assert.AreEqual(50, cleaner.MaxQueries);
Assert.AreEqual(200, cleaner.MaxValues);
Assert.AreEqual(2, cleaner.ReductionFactor);
Assert.AreEqual(500, cleaner.CleanFrequency);
Thread.Sleep(700);
}
// Check that no changes occurred
Assert.AreEqual(20, cache.EnumerateCache().Count());
totals = cache.Totals;
Assert.AreEqual(20, totals.TotalQueries);
Assert.AreEqual(80, totals.TotalValues);
Assert.IsNotNull(cache.Get(new object[] { objectName, "Idx1=2" }));
// Fill to max
for (int i = 20; i < 50; i++)
{
cache.Set(
new int[] { 1, 2, 3, 4 },
new object[] { objectName, "Idx" + i + "=2" });
}
Assert.AreEqual(50, cache.EnumerateCache().Count());
totals = cache.Totals;
Assert.AreEqual(50, totals.TotalQueries);
Assert.AreEqual(200, totals.TotalValues);
Assert.IsNotNull(cache.Get(new object[] { objectName, "Idx1=2" }));
using (var cleaner = new HardPruneCacheCleaner(cache,
50,
200,
2,
500)) // run every half second
{
Thread.Sleep(700);
}
Assert.AreEqual(50, cache.EnumerateCache().Count());
totals = cache.Totals;
Assert.AreEqual(50, totals.TotalQueries);
Assert.AreEqual(200, totals.TotalValues);
Assert.IsNotNull(cache.Get(new object[] { objectName, "Idx1=2" }));
// fill to 60 queries - with a cleanFactor of 2 the amount should be cut in half
for (int i = 50; i < 60; i++)
{
cache.Set(
new int[] { 1, 2, 3, 4 },
new object[] { objectName, "Idx" + i + "=2" });
}
Assert.AreEqual(60, cache.EnumerateCache().Count());
totals = cache.Totals;
Assert.AreEqual(60, totals.TotalQueries);
Assert.AreEqual(240, totals.TotalValues);
Assert.IsNotNull(cache.Get(new object[] { objectName, "Idx1=2" }));
using (var cleaner = new HardPruneCacheCleaner(cache,
50,
200,
2,
500)) // run every half second
{
Thread.Sleep(700);
}
// half of the queries should be gone
Assert.AreEqual(30, cache.EnumerateCache().Count());
totals = cache.Totals;
Assert.AreEqual(30, totals.TotalQueries);
Assert.AreEqual(120, totals.TotalValues);
Assert.IsNotNull(cache.Get(new object[] { objectName, "Idx1=2" }));
// Now overfill
for (int i = 100; i < 600; i++)
{
cache.Set(
new int[] { 1, 2, 3, 4 },
new object[] { objectName, "Idx" + i + "=2" });
}
Assert.AreEqual(530, cache.EnumerateCache().Count());
totals = cache.Totals;
Assert.AreEqual(530, totals.TotalQueries);
Assert.AreEqual(2120, totals.TotalValues);
Assert.IsNotNull(cache.Get(new object[] { objectName, "Idx1=2" }));
// this should cut out half
using (var cleaner = new HardPruneCacheCleaner(cache,
50,
200,
2,
500)) // run every half second
{
Thread.Sleep(700);
}
Assert.AreEqual(265, cache.EnumerateCache().Count());
totals = cache.Totals;
Assert.AreEqual(265, totals.TotalQueries);
Assert.AreEqual(1060, totals.TotalValues);
Assert.IsNotNull(cache.Get(new object[] { objectName, "Idx1=2" }));
// this should cut out another half
using (var cleaner = new HardPruneCacheCleaner(cache,
50,
200,
2,
500)) // run every half second
{
Thread.Sleep(700);
}
Assert.AreEqual(133, cache.EnumerateCache().Count());
totals = cache.Totals;
Assert.AreEqual(133, totals.TotalQueries);
Assert.AreEqual(532, totals.TotalValues);
Assert.IsNotNull(cache.Get(new object[] { objectName, "Idx1=2" }));
}
finally
{
cache.Dispose();
versions.Dispose();
metadata.Dispose();
}
}
public void CreateAndModifySameObjectMultithreaded()
{
Config config = ObjectTestHelper.GetConfigWithCaching();
ObjectMetadataStore metadata = new ObjectMetadataStore(config);
ObjectVersionStore versions = new ObjectVersionStore(config, metadata);
ObjectIndexerCache cache = new ObjectIndexerCache(metadata, versions);
string objectName = "TestObj";
string intVal1 = ObjectIndex.Create("IntIdx", 5).ToString();
string strVal1 = ObjectIndex.Create("StrIdx", "Val1").ToString();
object[] cacheParams = new object[] { objectName, intVal1, strVal1 };
int[] objectIds = new int[] { 1000, 2000 };
int nullGetCount = 0;
try
{
// Create a couple of threads that Get and Set a cache item.
// A third thread periodically expires the Object by changing its version.
List<Action> actions = new List<Action>();
for (int i = 0; i < 2; i++)
{
actions.Add(new Action(() =>
{
int localNullGetCount = 0;
for (int loops = 0; loops < 40; loops++)
{
for (int tryGet = 0; tryGet < 100; tryGet++)
{
int[] getObjectIds = cache.Get(cacheParams);
if (null == getObjectIds)
{
cache.Set(objectIds, cacheParams);
++localNullGetCount;
}
else
{
Assert.AreEqual(objectIds[0], getObjectIds[0]);
Assert.AreEqual(objectIds[1], getObjectIds[1]);
}
}
Thread.Sleep(100);
}
lock (objectIds)
{
nullGetCount += localNullGetCount;
}
}));
}
actions.Add(new Action(() =>
{
for (int loops = 0; loops < 40; loops++)
{
versions.Update(objectName);
Thread.Sleep(100);
}
}));
Parallel.Invoke(actions.ToArray());
Assert.IsTrue(nullGetCount > 2);
}
finally
{
cache.Dispose();
versions.Dispose();
metadata.Dispose();
}
}
public void CreateDefault()
{
Config config = ObjectTestHelper.GetConfigWithCaching();
ObjectMetadataStore metadata = new ObjectMetadataStore(config);
ObjectVersionStore versions = new ObjectVersionStore(config, metadata);
ObjectIndexerCache cache = new ObjectIndexerCache(metadata, versions);
using (var cleaner = new HardPruneCacheCleaner(cache))
{
try
{
Assert.AreEqual(HardPruneCacheCleaner.DefaultMaxQueries, cleaner.MaxQueries);
Assert.AreEqual(HardPruneCacheCleaner.DefaultMaxValues, cleaner.MaxValues);
Assert.AreEqual(HardPruneCacheCleaner.DefaultReductionFactor, cleaner.ReductionFactor);
Assert.AreEqual(HardPruneCacheCleaner.DefaultCleanFrequency, cleaner.CleanFrequency);
}
finally
{
cache.Dispose();
versions.Dispose();
metadata.Dispose();
}
}
}
public void CreateAndModifyMultithreaded()
{
Config config = ObjectTestHelper.GetConfigWithCaching();
ObjectMetadataStore metadata = new ObjectMetadataStore(config);
ObjectVersionStore versions = new ObjectVersionStore(config, metadata);
ObjectIndexerCache cache = new ObjectIndexerCache(metadata, versions);
try
{
// Create 4 actions to simulate 4 threads
// This test simulates 4 concurrent processes adding and reading items to the cache
// while they are also removed.
Action[] actions = new Action[4];
for (int i = 0; i < 4; i++)
{
string nextObjName = "Obj" + i;
actions[i] = new Action(() => {
for (int loops = 0; loops < 100; loops++)
{
object[][] cacheParams = new object[100][];
// create 100 cache entries
for (int add = 0; add < 100; add++)
{
cacheParams[add] = new object[] { nextObjName, "Idx" + ((loops * 100) + add) };
cache.Set(
new int[] { loops, loops + 1, loops + 2, loops + 3 },
cacheParams[add]);
}
// read 1000 entries
for (int read = 0; read < 1000; read++)
{
int[] getIds = cache.Get(cacheParams[read % 100]);
// may get removed so check for null
if (null != getIds)
{
Assert.AreEqual(loops, getIds[0]);
Assert.AreEqual(loops + 1, getIds[1]);
}
}
// Remove 20 entries - this can remove objects added by other threads
// Linq query is executed once write lock is aquired which is why this works
cache.Remove(cache.EnumerateCache().Take(20));
}
// that 100 times - total at end will equal 8000 entries and 32000 object ids
});
}
Parallel.Invoke(actions);
Assert.AreEqual(8000 * actions.Length, cache.EnumerateCache().Count());
CacheTotals totals = cache.Totals;
Assert.AreEqual(8000 * actions.Length, totals.TotalQueries);
Assert.AreEqual(32000 * actions.Length, totals.TotalValues);
cache.Reset();
Assert.AreEqual(0, cache.EnumerateCache().Count());
totals = cache.Totals;
Assert.AreEqual(0, totals.TotalQueries);
Assert.AreEqual(0, totals.TotalValues);
}
finally
{
cache.Dispose();
versions.Dispose();
metadata.Dispose();
}
}
public void CreateAndModify()
{
Config config = ObjectTestHelper.GetConfigWithCaching();
ObjectMetadataStore metadata = new ObjectMetadataStore(config);
ObjectVersionStore versions = new ObjectVersionStore(config, metadata);
ObjectIndexerCache cache = new ObjectIndexerCache(metadata, versions);
string objectName = "TestObj";
string intVal1 = ObjectIndex.Create("IntIdx", 5).ToString();
string strVal1 = ObjectIndex.Create("StrIdx", "Val1").ToString();
string strVal2 = ObjectIndex.Create("StrIdx", "Val3").ToString();
object[] cacheParams = new object[] { objectName, intVal1, strVal1 };
object[] cacheParams2 = new object[] { objectName, intVal1, strVal2 };
int[] objectIds = new int[] { 1000, 2000 };
int[] objectIds2 = new int[] { 3000, 4000 };
try
{
Assert.AreEqual(0, cache.EnumerateCache().Count());
CacheTotals totals = cache.Totals;
Assert.AreEqual(0, totals.TotalQueries);
Assert.AreEqual(0, totals.TotalValues);
Assert.IsNull(cache.Get(cacheParams));
cache.Reset();
Assert.AreEqual(0, cache.EnumerateCache().Count());
totals = cache.Totals;
Assert.AreEqual(0, totals.TotalQueries);
Assert.AreEqual(0, totals.TotalValues);
Assert.IsNull(cache.Get(cacheParams));
// add one query to cache
cache.Set(objectIds, cacheParams);
Assert.AreEqual(1, cache.EnumerateCache().Count());
totals = cache.Totals;
Assert.AreEqual(1, totals.TotalQueries);
Assert.AreEqual(2, totals.TotalValues);
int[] lookupIds = cache.Get(cacheParams);
Assert.AreEqual(objectIds.Length, lookupIds.Length);
Assert.AreEqual(objectIds[0], lookupIds[0]);
Assert.AreEqual(objectIds[1], lookupIds[1]);
cache.Reset();
Assert.AreEqual(0, cache.EnumerateCache().Count());
totals = cache.Totals;
Assert.AreEqual(0, totals.TotalQueries);
Assert.AreEqual(0, totals.TotalValues);
Assert.IsNull(cache.Get(cacheParams));
// add more than one query to cache
cache.Set(objectIds, cacheParams);
cache.Set(objectIds2, cacheParams2);
Assert.AreEqual(2, cache.EnumerateCache().Count());
totals = cache.Totals;
Assert.AreEqual(2, totals.TotalQueries);
Assert.AreEqual(4, totals.TotalValues);
lookupIds = cache.Get(cacheParams);
Assert.AreEqual(objectIds.Length, lookupIds.Length);
Assert.AreEqual(objectIds[0], lookupIds[0]);
Assert.AreEqual(objectIds[1], lookupIds[1]);
lookupIds = cache.Get(cacheParams2);
Assert.AreEqual(objectIds2.Length, lookupIds.Length);
Assert.AreEqual(objectIds2[0], lookupIds[0]);
Assert.AreEqual(objectIds2[1], lookupIds[1]);
// add the same query to cache again - make sure it replaces the existing one
cache.Set(objectIds2, cacheParams2);
Assert.AreEqual(2, cache.EnumerateCache().Count());
totals = cache.Totals;
Assert.AreEqual(2, totals.TotalQueries);
Assert.AreEqual(4, totals.TotalValues);
lookupIds = cache.Get(cacheParams);
Assert.AreEqual(objectIds.Length, lookupIds.Length);
Assert.AreEqual(objectIds[0], lookupIds[0]);
Assert.AreEqual(objectIds[1], lookupIds[1]);
lookupIds = cache.Get(cacheParams2);
Assert.AreEqual(objectIds2.Length, lookupIds.Length);
Assert.AreEqual(objectIds2[0], lookupIds[0]);
Assert.AreEqual(objectIds2[1], lookupIds[1]);
// remove query from cache
var entryToRemove = cache.EnumerateCache().Last();
// we don't know what order they will be returned so find out
// which value is being removed
object[] remainingCacheParams = cacheParams;
int[] remainingObjectIds = objectIds;
if (entryToRemove.Hash == ObjectIndexerCache.ConstructHash(cacheParams))
{
// then the first entry as removed so the second will be the
// remaining entry
remainingCacheParams = cacheParams2;
remainingObjectIds = objectIds2;
}
cache.Remove(new ICacheEntry[] { entryToRemove });
Assert.AreEqual(1, cache.EnumerateCache().Count());
totals = cache.Totals;
Assert.AreEqual(1, totals.TotalQueries);
Assert.AreEqual(2, totals.TotalValues);
lookupIds = cache.Get(remainingCacheParams);
Assert.AreEqual(remainingObjectIds.Length, lookupIds.Length);
Assert.AreEqual(remainingObjectIds[0], lookupIds[0]);
Assert.AreEqual(remainingObjectIds[1], lookupIds[1]);
Assert.IsNull(cache.Get(cacheParams2));
}
finally
{
cache.Dispose();
versions.Dispose();
metadata.Dispose();
}
}
public void CacheWithVersionChangeEvent()
{
Config config = ObjectTestHelper.GetConfigWithCaching();
ObjectMetadataStore metadata = new ObjectMetadataStore(config);
ObjectVersionStore versions = new ObjectVersionStore(config, metadata);
ObjectIndexerCache cache = new ObjectIndexerCache(metadata, versions);
string objectName = "TestObj";
string intVal1 = ObjectIndex.Create("IntIdx", 5).ToString();
string strVal1 = ObjectIndex.Create("StrIdx", "Val1").ToString();
string strVal2 = ObjectIndex.Create("StrIdx", "Val3").ToString();
object[] cacheParams = new object[] { objectName, intVal1, strVal1 };
object[] cacheParams2 = new object[] { objectName, intVal1, strVal2 };
int[] objectIds = new int[] { 1000, 2000 };
int[] objectIds2 = new int[] { 3000, 4000 };
try
{
// add a couple items to cache - they should not be returned
// once the object's version changes (they become "dirty")
cache.Set(objectIds, cacheParams);
cache.Set(objectIds2, cacheParams2);
Assert.AreEqual(2, cache.EnumerateCache().Count());
CacheTotals totals = cache.Totals;
Assert.AreEqual(2, totals.TotalQueries);
Assert.AreEqual(4, totals.TotalValues);
Assert.IsNotNull(cache.Get(cacheParams));
Assert.IsNotNull(cache.Get(cacheParams2));
versions.Update(objectName);
// NOTE: Dirty cache entries are not removed from cache as soon as
// they become dirty. They are removed lazily when Get/Set
// are called.
// CacheTotals and Enumerate will return Dirty objects and are
// only meant to be used by the Cache Cleaner and not directly
// by the Object Indexer.
Assert.AreEqual(2, cache.EnumerateCache().Count());
totals = cache.Totals;
Assert.AreEqual(2, totals.TotalQueries);
Assert.AreEqual(4, totals.TotalValues);
// Since the object is dirty, these should return null.
Assert.IsNull(cache.Get(cacheParams));
Assert.IsNull(cache.Get(cacheParams2));
// Reset the cache values and they should be returned again.
cache.Set(objectIds, cacheParams);
cache.Set(objectIds2, cacheParams2);
Assert.AreEqual(2, cache.EnumerateCache().Count());
totals = cache.Totals;
Assert.AreEqual(2, totals.TotalQueries);
Assert.AreEqual(4, totals.TotalValues);
Assert.IsNotNull(cache.Get(cacheParams));
Assert.IsNotNull(cache.Get(cacheParams2));
}
finally
{
cache.Dispose();
versions.Dispose();
metadata.Dispose();
}
}
public void CacheWithObjectRemovedEvent()
{
Config config = ObjectTestHelper.GetConfigWithCaching();
string ns = ObjectTestHelper.NameSpace1;
string obj = ObjectTestHelper.ObjectName1;
string objectFullName = ObjectNaming.CreateFullObjectName(ns, obj);
ObjectMetadata objectMetadata = new ObjectMetadata(ns, obj,
new ObjectIndexMetadata[]
{
new ObjectIndexMetadata("IntIndex1", ObjectIndexType.Integer),
new ObjectIndexMetadata("StrIndex1", ObjectIndexType.String, 15)
});
// Temporarily use the ObjectService to provision the Object's Metadata.
using (var svc = new ObjectService(config))
{
svc.CreateNameSpace(new ObjectNameSpaceConfig(ns,
"ZeroG Test", "Unit Test", DateTime.Now));
svc.ProvisionObjectStore(objectMetadata);
}
ObjectMetadataStore metadata = new ObjectMetadataStore(config);
ObjectVersionStore versions = new ObjectVersionStore(config, metadata);
ObjectIndexerCache cache = new ObjectIndexerCache(metadata, versions);
string intVal1 = ObjectIndex.Create("IntIdx", 5).ToString();
string strVal1 = ObjectIndex.Create("StrIdx", "Val1").ToString();
string strVal2 = ObjectIndex.Create("StrIdx", "Val3").ToString();
object[] cacheParams = new object[] { objectFullName, intVal1, strVal1 };
object[] cacheParams2 = new object[] { objectFullName, intVal1, strVal2 };
int[] objectIds = new int[] { 1000, 2000 };
int[] objectIds2 = new int[] { 3000, 4000 };
try
{
// add a couple items to cache - they should not be cleared
// when the object's metadata is removed
cache.Set(objectIds, cacheParams);
cache.Set(objectIds2, cacheParams2);
Assert.AreEqual(2, cache.EnumerateCache().Count());
CacheTotals totals = cache.Totals;
Assert.AreEqual(2, totals.TotalQueries);
Assert.AreEqual(4, totals.TotalValues);
Assert.IsNotNull(cache.Get(cacheParams));
Assert.IsNotNull(cache.Get(cacheParams2));
metadata.Remove(objectFullName);
// NOTE: Dirty cache entries are not removed from cache as soon as
// they become dirty. They are removed lazily when Get/Set
// are called.
// CacheTotals and Enumerate will return Dirty objects and are
// only meant to be used by the Cache Cleaner and not directly
// by the Object Indexer.
Assert.AreEqual(0, cache.EnumerateCache().Count());
totals = cache.Totals;
Assert.AreEqual(0, totals.TotalQueries);
Assert.AreEqual(0, totals.TotalValues);
Assert.IsNull(cache.Get(cacheParams));
Assert.IsNull(cache.Get(cacheParams2));
// The metadata needs to be added back so TestCleanup completes
metadata.StoreMetadata(objectMetadata);
}
finally
{
cache.Dispose();
versions.Dispose();
metadata.Dispose();
}
}
public void CacheWithDependencyVersionChangeEvent()
{
Config config = ObjectTestHelper.GetConfigWithCaching();
string ns = ObjectTestHelper.NameSpace1;
string obj = ObjectTestHelper.ObjectName1;
string obj2 = ObjectTestHelper.ObjectName2;
string objectFullName = ObjectNaming.CreateFullObjectName(ns, obj);
string objectFullName2 = ObjectNaming.CreateFullObjectName(ns, obj2);
ObjectMetadata objectMetadata = new ObjectMetadata(ns, obj,
new ObjectIndexMetadata[]
{
new ObjectIndexMetadata("IntIndex1", ObjectIndexType.Integer),
new ObjectIndexMetadata("StrIndex1", ObjectIndexType.String, 15)
});
ObjectMetadata objectMetadata2 = new ObjectMetadata(ns, obj2, null,
new string[] { obj }); // Make Object1 a dependency of Object2.
// Now whenever Object2's version changes, Object1's change event will fire as well.
// NOTE: Do not supply Full Object Name as objects can only depend on other objects
// within their namespace.
// Temporarily use the ObjectService to provision the Object's Metadata.
using (var svc = new ObjectService(config))
{
svc.CreateNameSpace(new ObjectNameSpaceConfig(ns,
"ZeroG Test", "Unit Test", DateTime.Now));
svc.ProvisionObjectStore(objectMetadata);
svc.ProvisionObjectStore(objectMetadata2);
}
ObjectMetadataStore metadata = new ObjectMetadataStore(config);
ObjectVersionStore versions = new ObjectVersionStore(config, metadata);
ObjectIndexerCache cache = new ObjectIndexerCache(metadata, versions);
string intVal1 = ObjectIndex.Create("IntIdx", 5).ToString();
string strVal1 = ObjectIndex.Create("StrIdx", "Val1").ToString();
string strVal2 = ObjectIndex.Create("StrIdx", "Val3").ToString();
object[] cacheParams = new object[] { objectFullName, intVal1, strVal1 };
object[] cacheParams2 = new object[] { objectFullName, intVal1, strVal2 };
int[] objectIds = new int[] { 1000, 2000 };
int[] objectIds2 = new int[] { 3000, 4000 };
try
{
// add a couple items to cache - they should not be returned
// once the objects dependency object version changes
cache.Set(objectIds, cacheParams);
cache.Set(objectIds2, cacheParams2);
Assert.AreEqual(2, cache.EnumerateCache().Count());
CacheTotals totals = cache.Totals;
Assert.AreEqual(2, totals.TotalQueries);
Assert.AreEqual(4, totals.TotalValues);
Assert.IsNotNull(cache.Get(cacheParams));
Assert.IsNotNull(cache.Get(cacheParams2));
versions.Update(objectFullName2);
Assert.AreEqual(2, cache.EnumerateCache().Count());
totals = cache.Totals;
Assert.AreEqual(2, totals.TotalQueries);
Assert.AreEqual(4, totals.TotalValues);
Assert.IsNull(cache.Get(cacheParams));
Assert.IsNull(cache.Get(cacheParams2));
cache.Set(objectIds, cacheParams);
cache.Set(objectIds2, cacheParams2);
Assert.AreEqual(2, cache.EnumerateCache().Count());
totals = cache.Totals;
Assert.AreEqual(2, totals.TotalQueries);
Assert.AreEqual(4, totals.TotalValues);
Assert.IsNotNull(cache.Get(cacheParams));
Assert.IsNotNull(cache.Get(cacheParams2));
}
finally
{
cache.Dispose();
versions.Dispose();
metadata.Dispose();
}
}