/// <summary>
///
/// </summary>
/// <param name="numProcessed"></param>
/// <returns></returns>
public bool PopAndUnlock(int numProcessed)
{
using (var disposable = new PooledRedisClientManager.DisposablePooledClient <SerializingRedisClient>(clientManager))
{
return(PopAndUnlock(numProcessed, disposable.Client));
}
}
public void DoneProcessedWorkItem()
{
numberOfProcessedItems++;
if (numberOfProcessedItems == numberOfDequeuedItems)
{
using (var disposable = new PooledRedisClientManager.DisposablePooledClient <SerializingRedisClient>(clientManager))
{
Unlock(disposable.Client);
}
}
}
/// <summary>
/// clear cache region
/// </summary>
public override void Clear()
{
//rename set of keys, and Start expiring the keys
using (var disposable = new PooledRedisClientManager.DisposablePooledClient<SerializingRedisClient>(ClientManager))
{
var client = disposable.Client;
using (var trans = client.CreateTransaction())
{
trans.QueueCommand(
r => r.IncrementValue(CacheNamespace.GetGenerationKey()), x => CacheNamespace.SetGeneration(x) );
var temp = "temp_" + CacheNamespace.GetGlobalKeysKey() + "_" + CacheNamespace.GetGeneration();
trans.QueueCommand(r => ((RedisNativeClient) r).Rename(CacheNamespace.GetGlobalKeysKey(), temp), null, e => Log.Debug(e) );
trans.QueueCommand(r => r.AddItemToList(RedisNamespace.NamespacesGarbageKey, temp));
trans.Commit();
}
}
}
/// <summary>
/// hit server for cache _region generation
/// </summary>
/// <returns></returns>
private long FetchGeneration()
{
using (var disposable = new PooledRedisClientManager.DisposablePooledClient<SerializingRedisClient>(ClientManager))
{
var generationKey = CacheNamespace.GetGenerationKey();
var client = disposable.Client;
var val = client.GetValue(generationKey);
return (val == null) ? client.Incr(generationKey) : Convert.ToInt64(val);
}
}
/// <summary>
/// Remove item corresponding to key from cache
/// </summary>
/// <param name="key"></param>
public override void Remove(object key)
{
if (key == null)
throw new ArgumentNullException("key");
if (Log.IsDebugEnabled)
Log.DebugFormat("removing item {0}", key);
IRedisPipeline pipe = null;
try
{
using (var disposable = new PooledRedisClientManager.DisposablePooledClient<SerializingRedisClient>(ClientManager))
{
var client = disposable.Client;
long generationFromServer = CacheNamespace.GetGeneration();
pipe = client.CreatePipeline();
pipe.QueueCommand(r => ((RedisNativeClient)r).Del(CacheNamespace.GlobalCacheKey(key)));
pipe.QueueCommand(r => r.GetValue(CacheNamespace.GetGenerationKey()),
x => generationFromServer = Convert.ToInt64(x));
pipe.Flush();
while (generationFromServer != CacheNamespace.GetGeneration())
{
//update cached generation value, and try again
CacheNamespace.SetGeneration(generationFromServer);
pipe.Replay();
}
}
}
catch (Exception)
{
Log.WarnFormat("could not delete key: {0}", key);
throw;
}
finally
{
if (pipe != null)
pipe.Dispose();
}
}
/// <summary>
/// Puts a LockedCacheableItem corresponding to (value, version) into
/// the cache
/// </summary>
/// <param name="putParameters"></param>
public void Put(IList<CacheVersionedPutParameters> putParameters)
{
//deal with null keys
IList<ScratchCacheItem> scratchItems = new List<ScratchCacheItem>();
foreach (var putParams in putParameters)
{
if (putParams.Key == null) continue;
scratchItems.Add(new ScratchCacheItem(putParams));
}
if (scratchItems.Count == 0) return;
byte[][] currentItemsRaw = null;
IRedisPipeline pipe = null;
try
{
using (var disposable = new PooledRedisClientManager.DisposablePooledClient<SerializingRedisClient>(ClientManager))
{
var client = disposable.Client;
long generationFromServer = CacheNamespace.GetGeneration();
pipe = client.CreatePipeline();
//watch for changes to generation key and cache key
IList<ScratchCacheItem> items = scratchItems;
pipe.QueueCommand(r => ((RedisClient)r).Watch(WatchKeys(items, true)));
//get all of the current objects
pipe.QueueCommand(r => ((RedisNativeClient)r).MGet(Keys(items, false)), x => currentItemsRaw = x);
pipe.QueueCommand(r => r.GetValue(CacheNamespace.GetGenerationKey()), x => generationFromServer = Convert.ToInt64(x));
pipe.Flush();
//make sure generation is correct before analyzing cache item
while (generationFromServer != CacheNamespace.GetGeneration())
{
//update cached generation value, and try again
CacheNamespace.SetGeneration(generationFromServer);
pipe.Replay();
}
// check if there is a new value to put
scratchItems = GenerateNewCacheItems(currentItemsRaw, scratchItems, client);
if (scratchItems.Count == 0)
return;
bool success;
// put new item in cache
using (var trans = client.CreateTransaction())
{
foreach (var scratch in scratchItems)
{
//setex on all new objects
ScratchCacheItem item = scratch;
trans.QueueCommand(
r =>
((IRedisNativeClient) r).SetEx(
CacheNamespace.GlobalCacheKey(item.PutParameters.Key),
_expiry, client.Serialize(item.NewCacheValue)));
//add keys to globalKeys set for this namespace
trans.QueueCommand(r => r.AddItemToSet(CacheNamespace.GetGlobalKeysKey(),
CacheNamespace.GlobalCacheKey(
item.PutParameters.Key)));
}
trans.QueueCommand(r => r.GetValue(CacheNamespace.GetGenerationKey()),
x => generationFromServer = Convert.ToInt64(x));
success = trans.Commit();
}
while (!success)
{
pipe.Replay();
//make sure generation is correct before analyzing cache item
while (generationFromServer != CacheNamespace.GetGeneration())
{
//update cached generation value, and try again
CacheNamespace.SetGeneration(generationFromServer);
pipe.Replay();
}
// check if there is a new value to put
scratchItems = GenerateNewCacheItems(currentItemsRaw, scratchItems, client);
if (scratchItems.Count == 0)
return;
// put new item in cache
using (var trans = client.CreateTransaction())
{
foreach (var scratch in scratchItems)
{
//setex on all new objects
ScratchCacheItem item = scratch;
trans.QueueCommand(
r =>
((IRedisNativeClient)r).SetEx(
CacheNamespace.GlobalCacheKey(item.PutParameters.Key),
_expiry, client.Serialize(item.NewCacheValue)));
//add keys to globalKeys set for this namespace
trans.QueueCommand(r => r.AddItemToSet(CacheNamespace.GetGlobalKeysKey(),
CacheNamespace.GlobalCacheKey(
item.PutParameters.Key)));
}
trans.QueueCommand(r => r.GetValue(CacheNamespace.GetGenerationKey()),
x => generationFromServer = Convert.ToInt64(x));
success = trans.Commit();
}
}
// if we get here, we know that the generation has not been changed
// otherwise, the WATCH would have failed the transaction
CacheNamespace.SetGeneration(generationFromServer);
}
}
catch (Exception)
{
foreach (var putParams in putParameters)
{
Log.WarnFormat("could not get: {0}", putParams.Key);
}
throw;
}
finally
{
if (pipe != null)
pipe.Dispose();
}
}
/// <summary>
///
/// </summary>
/// <param name="key"></param>
/// <param name="value"></param>
public void Put(CachePutParameters putParameters)
{
var key = putParameters.Key;
var value = putParameters.Value;
if (key == null)
throw new ArgumentNullException("key", "null key not allowed");
if (value == null)
throw new ArgumentNullException("value", "null value not allowed");
if (Log.IsDebugEnabled)
Log.DebugFormat("setting value for item {0}", key);
try
{
using (var disposable = new PooledRedisClientManager.DisposablePooledClient<SerializingRedisClient>(ClientManager))
{
var client = disposable.Client;
var bytes = client.Serialize(value);
//do transactioned get of generation and value
//if it succeeds, and null is returned, then either the key doesn't exist or
// our generation is out of date. In the latter case , update generation and try
// again.
long generationFromServer = CacheNamespace.GetGeneration();
while (true)
{
using (var trans = client.CreateTransaction())
{
trans.QueueCommand(r => ((IRedisNativeClient)r).SetEx(CacheNamespace.GlobalCacheKey(key),
_expiry, bytes));
//add key to globalKeys set for this namespace
trans.QueueCommand(r => r.AddItemToSet(CacheNamespace.GetGlobalKeysKey(),
CacheNamespace.GlobalCacheKey(key)));
trans.QueueCommand(r => r.GetValue(CacheNamespace.GetGenerationKey()),
x => generationFromServer = Convert.ToInt64(x));
trans.Commit();
}
if (generationFromServer != CacheNamespace.GetGeneration())
{
//update cached generation value, and try again
CacheNamespace.SetGeneration(generationFromServer);
}
else
break;
}
}
}
catch (Exception)
{
Log.WarnFormat("could not get: {0}", key);
throw;
}
}
/// <summary>
/// Return a dictionary of (key,value) pairs corresponding to a collection of keys
/// </summary>
/// <param name="keys"></param>
/// <returns></returns>
public override IDictionary MultiGet(IEnumerable keys)
{
var rc = new Dictionary<object, object>();
using (var disposable = new PooledRedisClientManager.DisposablePooledClient<SerializingRedisClient>(ClientManager))
{
var client = disposable.Client;
long generationFromServer = CacheNamespace.GetGeneration();
byte[][] resultBytesArray = null;
var keyCount = 0;
using (var pipe = client.CreatePipeline())
{
var globalKeys = new List<string>();
//generate global keys
foreach (var key in keys)
{
keyCount++;
globalKeys.Add(CacheNamespace.GlobalCacheKey(key));
}
pipe.QueueCommand(r => ((RedisNativeClient) r).MGet(globalKeys.ToArray()),
x => resultBytesArray = x);
pipe.QueueCommand(r => r.GetValue(CacheNamespace.GetGenerationKey()),
x => generationFromServer = Convert.ToInt64(x));
pipe.Flush();
}
while (generationFromServer != CacheNamespace.GetGeneration())
{
//update cached generation value, and try again
CacheNamespace.SetGeneration(generationFromServer);
using (var pipe = client.CreatePipeline())
{
var globalKeys = new List<string>();
//generate global keys
keyCount = 0;
foreach (var key in keys)
{
keyCount++;
globalKeys.Add(CacheNamespace.GlobalCacheKey(key));
}
pipe.QueueCommand(r => ((RedisNativeClient)r).MGet(globalKeys.ToArray()),
x => resultBytesArray = x);
pipe.QueueCommand(r => r.GetValue(CacheNamespace.GetGenerationKey()),
x => generationFromServer = Convert.ToInt64(x));
pipe.Flush();
}
}
if (keyCount != resultBytesArray.Length)
throw new RedisException("MultiGet: number of results does not match number of keys");
//process results
var iter = keys.GetEnumerator();
iter.MoveNext();
foreach (var resultBytes in resultBytesArray)
{
if (resultBytes != null)
{
var currentObject = client.Deserialize(resultBytes);
if (currentObject != null)
rc[iter.Current] = currentObject;
}
iter.MoveNext();
}
return rc;
}
}
/// <summary>
///
/// </summary>
/// <param name="key"></param>
public override void Unlock(object key)
{
if (!AcquiredLocks.ContainsKey(key))
return;
using (var disposable = new PooledRedisClientManager.DisposablePooledClient<SerializingRedisClient>(ClientManager))
{
var aLock = new DistributedLock();
var globalKey = CacheNamespace.GlobalKey(key, RedisNamespace.NumTagsForLockKey);
if (aLock.Unlock(globalKey, AcquiredLocks[key], disposable.Client))
AcquiredLocks.Remove(key);
}
}
/// <summary>
///
/// </summary>
/// <param name="key"></param>
public override void Unlock(object key)
{
if (!AcquiredLocks.ContainsKey(key))
return;
using (var disposable = new PooledRedisClientManager.DisposablePooledClient<SerializingRedisClient>(ClientManager))
{
disposable.Client.Unlock(CacheNamespace.GlobalKey(key, RedisNamespace.NumTagsForLockKey), AcquiredLocks[key]);
AcquiredLocks.Remove(key);
}
}
/// <summary>
/// Remove item corresponding to key from cache
/// </summary>
/// <param name="key"></param>
public override void Remove(object key)
{
if (key == null)
throw new ArgumentNullException("key");
if (Log.IsDebugEnabled)
Log.DebugFormat("removing item {0}", key);
using (var disposable = new PooledRedisClientManager.DisposablePooledClient<SerializingRedisClient>(ClientManager))
{
var client = disposable.Client;
using (var pipe = client.CreatePipeline())
{
//watch for changes to cache keys
pipe.QueueCommand(r => ((RedisNativeClient)r).Del(CacheNamespace.GlobalCacheKey(key)));
pipe.Flush();
}
}
}
/// <summary>
/// Puts a LockedCacheableItem corresponding to (value, version) into
/// the cache
/// </summary>
/// <param name="putParameters"></param>
public void Put(IList<CacheVersionedPutParameters> putParameters)
{
//deal with null keys
IList<ScratchCacheItem> scratchItems = new List<ScratchCacheItem>();
foreach (var putParams in putParameters)
{
if (putParams.Key == null) continue;
scratchItems.Add(new ScratchCacheItem(putParams));
if (Log.IsDebugEnabled)
Log.DebugFormat("fetching object {0} from the cache", putParams.Key.ToString());
}
if (scratchItems.Count == 0) return;
byte[][] currentItemsRaw = null;
IRedisPipeline pipe = null;
try
{
using (var disposable = new PooledRedisClientManager.DisposablePooledClient<SerializingRedisClient>(ClientManager))
{
var client = disposable.Client;
pipe = client.CreatePipeline();
//watch for changes to cache keys
IList<ScratchCacheItem> items = scratchItems;
pipe.QueueCommand(r => ((RedisClient)r).Watch(WatchKeys(items)));
//get all of the current objects
pipe.QueueCommand(r => ((RedisNativeClient)r).MGet(Keys(items)), x => currentItemsRaw = x);
pipe.Flush();
// check if there is are new cache items to put
scratchItems = GenerateNewCacheItems(currentItemsRaw, scratchItems, client);
if (scratchItems.Count == 0)
return;
bool success;
// try to put new items in cache
using (var trans = client.CreateTransaction())
{
foreach (var scratch in scratchItems)
{
//setex on all new objects
ScratchCacheItem item = scratch;
trans.QueueCommand(
r => ((IRedisNativeClient) r).SetEx(
CacheNamespace.GlobalCacheKey(item.PutParameters.Key),
_expiry, client.Serialize(item.NewCacheValue) ));
}
success = trans.Commit();
}
while (!success)
{
pipe.Replay();
// check if there is a new value to put
scratchItems = GenerateNewCacheItems(currentItemsRaw, scratchItems, client);
if (scratchItems.Count == 0)
return;
// try to put new items in cache
using (var trans = client.CreateTransaction())
{
foreach (var scratch in scratchItems)
{
//setex on all new objects
ScratchCacheItem item = scratch;
trans.QueueCommand(
r =>
((IRedisNativeClient)r).SetEx(
CacheNamespace.GlobalCacheKey(item.PutParameters.Key),
_expiry, client.Serialize(item.NewCacheValue) ));
}
//update live query cache
success = trans.Commit();
}
}
}
}
catch (Exception)
{
foreach (var putParams in putParameters)
{
Log.WarnFormat("could not get: {0}", putParams.Key);
}
throw;
}
finally
{
if (pipe != null)
pipe.Dispose();
}
}
/// <summary>
///
/// </summary>
/// <param name="putParameters"></param>
public void Put(CachePutParameters putParameters)
{
var key = putParameters.Key;
var value = putParameters.Value;
if (key == null)
throw new ArgumentNullException("key", "null key not allowed");
if (value == null)
throw new ArgumentNullException("value", "null value not allowed");
if (Log.IsDebugEnabled)
Log.DebugFormat("setting value for item {0}", key);
try
{
using (var disposable = new PooledRedisClientManager.DisposablePooledClient<SerializingRedisClient>(ClientManager))
{
var client = disposable.Client;
var globalKey = CacheNamespace.GlobalCacheKey(key);
using (var pipe = client.CreatePipeline())
{
pipe.QueueCommand(r => ((IRedisNativeClient)r).SetEx(globalKey, _expiry, client.Serialize(value)));
pipe.Flush();
}
}
}
catch (Exception)
{
Log.WarnFormat("could not put {0} for key {1}", value, key);
throw;
}
}
/// <summary>
/// Return a dictionary of (key,value) pairs corresponding to a collection of keys
/// </summary>
/// <param name="keys"></param>
/// <returns></returns>
public override IDictionary MultiGet(IEnumerable keys)
{
var rc = new Dictionary<object, object>();
using (var disposable = new PooledRedisClientManager.DisposablePooledClient<SerializingRedisClient>(ClientManager))
{
var client = disposable.Client;
var globalKeys = new List<string>();
//generate global keys
var keyCount = 0;
foreach (var key in keys)
{
keyCount++;
globalKeys.Add(CacheNamespace.GlobalCacheKey(key));
}
// do multi get
var resultBytesArray = client.MGet(globalKeys.ToArray());
if (keyCount != resultBytesArray.Length)
throw new RedisException("MultiGet: number of results does not match number of keys");
//process results
var iter = keys.GetEnumerator();
iter.MoveNext();
foreach (var resultBytes in resultBytesArray)
{
if (resultBytes != null)
{
var currentObject = client.Deserialize(resultBytes);
if (currentObject != null)
rc[iter.Current] = currentObject;
}
iter.MoveNext();
}
}
return rc;
}
/// <summary>
///
/// </summary>
/// <param name="key"></param>
public override void Lock(object key)
{
using (var disposable = new PooledRedisClientManager.DisposablePooledClient<SerializingRedisClient>(ClientManager))
{
long lockExpire = disposable.Client.Lock(CacheNamespace.GlobalKey(key, RedisNamespace.NumTagsForLockKey), _lockAcquisitionTimeout, _lockTimeout);
bool rc = (lockExpire != 0);
if (rc)
AcquiredLocks[key] = lockExpire;
}
}
/// <summary>
///
/// </summary>
/// <param name="key"></param>
/// <returns></returns>
public override object Get(object key)
{
if (key == null)
return null;
if (Log.IsDebugEnabled)
Log.DebugFormat("fetching object {0} from the cache", key);
object rc;
try
{
using (var disposable = new PooledRedisClientManager.DisposablePooledClient<SerializingRedisClient>(ClientManager))
{
var client = disposable.Client;
var maybeObj = client.Get(CacheNamespace.GlobalCacheKey(key));
rc = (maybeObj == null) ? null : client.Deserialize(maybeObj);
}
}
catch (Exception)
{
Log.WarnFormat("could not get: {0}", key);
throw;
}
return rc;
}
/// <summary>
///
/// </summary>
/// <param name="key"></param>
/// <returns></returns>
public override object Get(object key)
{
if (key == null)
return null;
if (Log.IsDebugEnabled)
Log.DebugFormat("fetching object {0} from the cache", key);
byte[] maybeObj = null;
object rc;
IRedisPipeline pipe = null;
try
{
using (var disposable = new PooledRedisClientManager.DisposablePooledClient<SerializingRedisClient>(ClientManager))
{
var client = disposable.Client;
//do transactioned get of generation and value
//if it succeeds, and null is returned, then either the key doesn't exist or
// our generation is out of date. In the latter case , update generation and try
// again.
long generationFromServer = CacheNamespace.GetGeneration();
pipe = client.CreatePipeline();
pipe.QueueCommand(r => ((RedisNativeClient) r).Get(CacheNamespace.GlobalCacheKey(key)),
x => maybeObj = x);
pipe.QueueCommand(r => r.GetValue(CacheNamespace.GetGenerationKey()),
x => generationFromServer = Convert.ToInt64(x));
pipe.Flush();
while (generationFromServer != CacheNamespace.GetGeneration())
{
//update cached generation value, and try again
CacheNamespace.SetGeneration(generationFromServer);
pipe.Replay();
}
rc = client.Deserialize(maybeObj);
}
}
catch (Exception)
{
Log.WarnFormat("could not get: {0}", key);
throw;
}
finally
{
if (pipe != null)
pipe.Dispose();
}
return rc;
}
/// <summary>
///
/// </summary>
/// <param name="key"></param>
public override void Lock(object key)
{
using (var disposable = new PooledRedisClientManager.DisposablePooledClient<SerializingRedisClient>(ClientManager))
{
long lockExpire = 0;
var aLock = new DistributedLock();
var globalKey = CacheNamespace.GlobalKey(key, RedisNamespace.NumTagsForLockKey);
try
{
long rc = aLock.Lock(globalKey, _lockAcquisitionTimeout, _lockTimeout, out lockExpire, disposable.Client);
if (rc != DistributedLock.LOCK_NOT_ACQUIRED)
AcquiredLocks[key] = lockExpire;
}
catch (Exception)
{
if (lockExpire != 0)
aLock.Unlock(globalKey, lockExpire, disposable.Client);
}
}
}
