public static bool ConcurrentIncreaseIfConditionFromDistributedCache(string key, int change, Func <int, bool> valueAfterChangeCondition, out int increasedValue) { if (change == 0) { var t = MemcachedCacheHelper.GetClientInstance().Get(key); if (t == null || t.GetType() != typeof(int)) { increasedValue = 0; } else { increasedValue = (int)t; } return(true); } if (valueAfterChangeCondition == null) { throw new ArgumentNullException("changedValueCondition"); } return(ConcurrentUpdateFromDistributedCache(key, (int d, out bool need) => { need = true; d = d + change; if (valueAfterChangeCondition != null && valueAfterChangeCondition(d) == false) { need = false; } return d; }, out increasedValue)); }
public MemcachedQueueItemProcessResult SyncGetMessageProcessResult(int timeoutMilliseconds = 10000) { if (EnqueueSuccess == false || MessageItemIndex <= 0) { return(null); } var queueId = MemcachedQueue.GetQueueId(m_QueueName); if (string.IsNullOrWhiteSpace(queueId)) { return(null); } string itemKey = MemcachedQueue.BuildItemIndexKeyForProcessResult(queueId, (ulong)MessageItemIndex); var client = MemcachedCacheHelper.GetClientInstance(); int wait = 0; while (wait < timeoutMilliseconds) { var r = client.Get(itemKey) as MemcachedQueueItemProcessResult; if (r != null) { return(r); } Thread.Sleep(100); wait = wait + 100; } return(null); }
public static MemcachedQueueEnqueueResult Enqueue(string queueName, object data, int generation = 0, uint maxRetryTimes = 2) { if (maxRetryTimes < 0) { maxRetryTimes = 0; } if (string.IsNullOrWhiteSpace(queueName)) { throw new ArgumentNullException("queueName"); } queueName = queueName.Trim(); string localCacheKey = "MemQ.Id|" + queueName; bool rst = false; var client = MemcachedCacheHelper.GetClientInstance(); ulong? index; string queueId; bool notReadFromLocalCache = false; do { queueId = GetOrCreateQueueId(queueName, notReadFromLocalCache); index = client.Increment(BuildIndexKey(queueId, MemcachedQueueIndexType.Enqueue), 1); if (index == null) // 有可能本地缓存读取的id已经过期试下了,需要在下次循环执行时,不走本地缓存重新获取 { notReadFromLocalCache = true; } }while (index == null); // 程序执行到这里,EnqueueIndex已经被增长上去了,但此index处的数据还未Set写入,那么此时并发的Dequeue操作则有可能读取到此index的,但获取数据是获取不到的 // 所以在Dequeue方法里有针对这种极端并发的处理代码,结合这里的代码注释就容易理解了 string itemKey = BuildItemIndexKey(queueId, index.Value); var item = new MemcachedQueueItem { Data = data, Generation = generation }; rst = client.Set(itemKey, item); // 程序执行到这里才真正开始将此index的数据写入 int retryTime = 0; while (rst == false) // 如果写入失败,则重试 { if (retryTime >= maxRetryTimes) // 重试次数达到最大重试次数,则不再重试 { break; } Thread.Sleep(10); // 等待10ms再试 rst = client.Set(itemKey, item); retryTime++; } return(new MemcachedQueueEnqueueResult(queueName, rst, index.Value)); }
public static MemcachedQueueItem[] GetAllItems(string queueName) { if (string.IsNullOrWhiteSpace(queueName)) { throw new ArgumentNullException("queueName"); } queueName = queueName.Trim(); string queueId = GetQueueId(queueName); if (string.IsNullOrWhiteSpace(queueId)) // 队列列表里未找到指定名称的队列 { return(new MemcachedQueueItem[0]); } var ary = GetIndexById(queueId, MemcachedQueueIndexType.Dequeue, MemcachedQueueIndexType.Enqueue); ulong de_index = ary == null || ary.Length <= 0 ? 0 : ary[0]; ulong en_index = ary == null || ary.Length <= 1 ? 0 : ary[1]; if (en_index <= de_index) // 说明队列是空的 { return(new MemcachedQueueItem[0]); } ulong len = en_index - de_index; MemcachedQueueItem[] rstList = new MemcachedQueueItem[len]; const ulong maxBatchSize = 100; ulong itemIndex = de_index; int index = 0; var client = MemcachedCacheHelper.GetClientInstance(); while (itemIndex < en_index) { ulong leftItemCount = en_index - itemIndex; uint batchSize = (uint)(leftItemCount < maxBatchSize ? leftItemCount : maxBatchSize); string[] keys = new string[batchSize]; for (uint i = 0; i < batchSize; i++) { itemIndex++; keys[i] = BuildItemIndexKey(queueId, itemIndex); } var objs = client.Get(keys); for (uint i = 0; i < batchSize; i++) { var obj = (objs == null || i >= objs.Length) ? null : objs[i]; rstList[index] = obj as MemcachedQueueItem; index++; } } return(rstList); }
public static Dictionary <string, string> GetAllQueues() { var client = MemcachedCacheHelper.GetClientInstance(); var rst = client.Get(M_QUEUE_REGISTRY_KEY) as Dictionary <string, int>; if (rst == null) { return(null); } Dictionary <string, string> dicts = new Dictionary <string, string>(rst.Count * 2); foreach (var entry in rst) { dicts.Add(entry.Key, entry.Value.ToString(CultureInfo.InvariantCulture)); } return(dicts); }
public static int ConcurrentIncreaseFromDistributedCache(string key, int change) { if (change == 0) { var t = MemcachedCacheHelper.GetClientInstance().Get(key); if (t == null || t.GetType() != typeof(int)) { return(0); } return((int)t); } int v; ConcurrentUpdateFromDistributedCache(key, (int d, out bool need) => { need = true; return(d + change); }, out v); return(v); }
// 此方法不用考虑线程安全 private static void ClearGarbageItemInQueue(string queueId) { var ary = GetIndexById(queueId, MemcachedQueueIndexType.Dequeue, MemcachedQueueIndexType.ClearGarbage); ulong de_index = ary == null || ary.Length <= 0 ? 0 : ary[0]; ulong gc_index = ary == null || ary.Length <= 1 ? 0 : ary[1]; if (gc_index >= de_index - 1) // 垃圾回收到的地方,已经是最近一次读取的地方的前一个地方了,说明垃圾已经回收完了 { return; } var client = MemcachedCacheHelper.GetClientInstance(); for (var i = gc_index + 1; i < de_index; i++) // 不能等于de_index,这样确保当前最近处理的item数据还在 { string itemKey = BuildItemIndexKey(queueId, i); //string resultKey = BuildItemIndexKeyForProcessResult(queueId, i); client.Delete(itemKey); } client.Set(BuildIndexKey(queueId, MemcachedQueueIndexType.ClearGarbage), de_index.ToString(CultureInfo.InvariantCulture)); }
private static ulong GetIndexById(string queueId, MemcachedQueueIndexType indexType) { if (string.IsNullOrWhiteSpace(queueId)) // 队列列表里未找到指定名称的队列 { return(0); } string indexKey = BuildIndexKey(queueId, indexType); var client = MemcachedCacheHelper.GetClientInstance(); string index_string = client.Get(indexKey) as string; ulong index; if (string.IsNullOrWhiteSpace(index_string) || index_string == "0" || ulong.TryParse(index_string, out index) == false || index <= 0) { return(0); } return(index); }
public static MemcachedQueueItem GetItemWithProcessResult(string queueName, long index, out MemcachedQueueItemProcessResult processResult) { if (string.IsNullOrWhiteSpace(queueName)) { throw new ArgumentNullException("queueName"); } if (index <= 0) { processResult = null; return(null); } queueName = queueName.Trim(); var queueId = GetQueueId(queueName); if (string.IsNullOrWhiteSpace(queueId)) { processResult = null; return(null); } string itemKey = BuildItemIndexKey(queueId, (ulong)index); string itemProcessdResultKey = BuildItemIndexKeyForProcessResult(queueId, (ulong)index); var client = MemcachedCacheHelper.GetClientInstance(); var objs = client.Get(new string[] { itemKey, itemProcessdResultKey }); MemcachedQueueItem item = null; processResult = null; if (objs != null) { if (objs.Length > 0) { item = objs[0] as MemcachedQueueItem; } if (objs.Length > 1) { processResult = objs[1] as MemcachedQueueItemProcessResult; } } return(item); }
public static MemcachedQueueItem GetItem(string queueName, long index) { if (string.IsNullOrWhiteSpace(queueName)) { throw new ArgumentNullException("queueName"); } if (index <= 0) { return(null); } queueName = queueName.Trim(); var queueId = GetQueueId(queueName); if (string.IsNullOrWhiteSpace(queueId)) { return(null); } string itemKey = BuildItemIndexKey(queueId, (ulong)index); var client = MemcachedCacheHelper.GetClientInstance(); return(client.Get(itemKey) as MemcachedQueueItem); }
public static MemcachedQueueItem[] GetItem(string queueName, long[] indexs) { if (string.IsNullOrWhiteSpace(queueName)) { throw new ArgumentNullException("queueName"); } if (indexs == null || indexs.Length <= 0) { return(new MemcachedQueueItem[0]); } queueName = queueName.Trim(); var queueId = GetQueueId(queueName); if (string.IsNullOrWhiteSpace(queueId)) { return(new MemcachedQueueItem[indexs.Length]); } string[] itemKeys = new string[indexs.Length]; for (var i = 0; i < indexs.Length; i++) { itemKeys[i] = BuildItemIndexKey(queueId, indexs[i] <= 0 ? 0 : (ulong)indexs[i]); } var client = MemcachedCacheHelper.GetClientInstance(); var list = client.Get(itemKeys); if (list == null || list.Length <= 0) { return(new MemcachedQueueItem[indexs.Length]); } MemcachedQueueItem[] rstList = new MemcachedQueueItem[indexs.Length]; for (uint i = 0; i < indexs.Length; i++) { var obj = (list == null || i >= list.Length) ? null : list[i]; rstList[i] = obj as MemcachedQueueItem; } return(rstList); }
private static ulong[] GetIndexById(string queueId, params MemcachedQueueIndexType[] indexTypes) { if (string.IsNullOrWhiteSpace(queueId)) // 队列列表里未找到指定名称的队列 { return(new ulong[indexTypes.Length]); } string[] indexKeys = new string[indexTypes.Length]; for (int i = 0; i < indexTypes.Length; i++) { indexKeys[i] = BuildIndexKey(queueId, indexTypes[i]); } var client = MemcachedCacheHelper.GetClientInstance(); object[] rstArray = client.Get(indexKeys); if (rstArray == null || rstArray.Length <= 0) { return(new ulong[0]); } ulong[] rstList = new ulong[rstArray.Length]; for (int i = 0; i < rstArray.Length; i++) { string index_string = rstArray[i] as string; ulong index; if (string.IsNullOrWhiteSpace(index_string) || index_string == "0" || ulong.TryParse(index_string, out index) == false || index <= 0) { rstList[i] = 0; } else { rstList[i] = index; } } return(rstList); }
public static bool Dequeue <T>(string queueName, MemcachedQueueItemHandler <T> handler, uint maxRetryTimes = 2) { if (maxRetryTimes < 0) { maxRetryTimes = 0; } if (string.IsNullOrWhiteSpace(queueName)) { throw new ArgumentNullException("queueName"); } if (CheckSerializable(typeof(T)) == false) { throw new NotSupportedException("不支持handler消息处理返回结果的类型\"" + typeof(T).FullName + "\",因为此类型不支持二进制序列化,如果要使用请在此类型的定义处添加[Serializable]的Attribute"); } queueName = queueName.Trim(); string queueId = GetQueueId(queueName); if (string.IsNullOrWhiteSpace(queueId)) // 队列列表里未找到指定名称的队列 { //handler(-1, null); return(false); } ulong en_index = GetIndexById(queueId, MemcachedQueueIndexType.Enqueue); if (en_index <= 0) // 还从未写入过,那么肯定读取不到数据了 { //handler(-1, null); return(false); } var client = MemcachedCacheHelper.GetClientInstance(); MemcachedClient.CasResult casRst; ulong unique; string k1 = BuildIndexKey(queueId, MemcachedQueueIndexType.Dequeue); ulong de_index; MemcachedQueueItem item; do { string de_index_string = client.Gets(k1, out unique) as string; // 读取上次出队列的index if (string.IsNullOrWhiteSpace(de_index_string) || de_index_string == "0" || ulong.TryParse(de_index_string, out de_index) == false || de_index <= 0) { // 说明之前还从未出过队列 de_index = 0; } de_index++; // 计算出本次应该出队列的item的index if (de_index > en_index) // 如果已经超过队列最近入的index了,说明队列已经读空了(所有item都读取过了) { //handler(-1, null); return(false); } string itemKey = BuildItemIndexKey(queueId, de_index); item = client.Get(itemKey) as MemcachedQueueItem; // 这里可能遇到一种极端的读写并发情况,即在Enqueue方法的地方先将EnqueueIndex增长上去了,但此index处的元素数据还未来得及Set写 // 然后本Dequeue方法就先拿到此index,然后Get数据,结果因为Set还未来得及写入所以获取数据未空 // 所以需要等待一会儿再试(期望这段等待时间内写入线程能完成此index处的数据的写入) int reTryTimes = 0; while (item == null) { Thread.Sleep(10); // 等待10ms再试着获取此index处的数据 item = client.Get(itemKey) as MemcachedQueueItem; reTryTimes++; if (reTryTimes >= maxRetryTimes) // 如果重试达到maxRetryTimes次(累计maxRetryTimes * 10ms了),那么基本上证明不是并发还未写入了,因为并发写入问题不可能等这么久还未写入进去,基本上确定是遇到网络问题或脏数据,或者memcached服务挂了 { break; } // 基本上Dequeue操作都是单线程慢慢处理,所以对执行时间要求不是那么高,500ms的延迟等待可以接受(并且是要遇到item==null的很极端的读写并发,或者是遇到了脏数据,发生的几率都很小) } if (unique == 0) // 说明还不存在,需要新增 { casRst = client.Add(k1, de_index.ToString(CultureInfo.InvariantCulture)) ? MemcachedClient.CasResult.Stored : MemcachedClient.CasResult.NotStored; } else { casRst = client.CheckAndSet(k1, de_index.ToString(CultureInfo.InvariantCulture), unique); // 乐观锁,满足条件(没有人优先改动过)则更新缓存,如果更新成功则锁定了de_index这个索引值,其它线程无法再获取到这个index的item了 } } while ( casRst != MemcachedClient.CasResult.Stored || // 乐观锁,这种写法防止并发冲突 item == null // 发生item == null这种情况可能是脏数据了,则继续找下一个索引处的元素 ); if (item == null) // 说明没有找到有效数据 { //handler(-1, null); return(false); } string itemProcessdResultKey = BuildItemIndexKeyForProcessResult(queueId, de_index); T rst = default(T); Exception ex = null; try { rst = handler((long)de_index, item); ex = null; } catch (Exception ex1) { rst = default(T); ex = ex1; throw; } finally { MemcachedQueueItemProcessResult pr = new MemcachedQueueItemProcessResult { Exception = ex, Result = rst }; client.Set(itemProcessdResultKey, pr, DateTime.Now.AddMinutes(120)); } return(true); }
public static string GetOrCreateQueueId(string queueName, bool notReadFromLocalCache = false) { if (string.IsNullOrWhiteSpace(queueName)) { throw new ArgumentNullException("queueName"); } queueName = queueName.Trim(); string localCacheKey = BuildLocalCacheKeyOfQueueId(queueName); Func <string> getter = () => { var client = MemcachedCacheHelper.GetClientInstance(); MemcachedClient.CasResult casRst; ulong unique; int queueId; string enKey = null; do { Dictionary <string, int> registryDicts = client.Gets(M_QUEUE_REGISTRY_KEY, out unique) as Dictionary <string, int>; // 获取队列注册表 if (registryDicts != null && registryDicts.Count > 0 && registryDicts.TryGetValue(queueName, out queueId)) { // 说明队列注册表里找到了此queueName的队列,获取到queue的Id(可能是之前注册过的,或者并发的其它线程先注册到了) break; } // 注册表里没找到,则准备为此queueName的队列生成一个新的唯一Id,注册到注册表里 queueId = (registryDicts == null || registryDicts.Count <= 0) ? 1 : registryDicts.Values.Max() + 1; if (registryDicts == null) { registryDicts = new Dictionary <string, int> { { queueName, queueId } }; } else { if (registryDicts.ContainsKey(queueName)) { registryDicts[queueName] = queueId; } else { registryDicts.Add(queueName, queueId); } } enKey = BuildIndexKey(queueId.ToString(CultureInfo.InvariantCulture), MemcachedQueueIndexType.Enqueue); client.Add(enKey, "0"); if (unique == 0) // 说明还不存在,需要新增 { casRst = client.Add(M_QUEUE_REGISTRY_KEY, registryDicts) ? MemcachedClient.CasResult.Stored : MemcachedClient.CasResult.NotStored; } else { casRst = client.CheckAndSet(M_QUEUE_REGISTRY_KEY, registryDicts, unique); // 满足条件则更新缓存 } } while (casRst != MemcachedClient.CasResult.Stored); // 这种写法防止并发冲突 return(queueId.ToString(CultureInfo.InvariantCulture)); }; if (notReadFromLocalCache) { string id = getter(); if (string.IsNullOrWhiteSpace(id) == false) { Cache.SetLocalCache(localCacheKey, id, true, QUEUE_ID_LOCAL_CACHE_EXPIRY_MIN); } return(id); } return(Cache.GetWithLocalCache(localCacheKey, getter, true, QUEUE_ID_LOCAL_CACHE_EXPIRY_MIN)); }