/// <summary> /// 放到消息队列池中 /// </summary> /// <param name="statement"></param> public static bool Put(SqlStatement statement) { bool result = false; try { if (!IsUseSyncQueue) { return(false); } string tableName = statement.Table; string key = GetSqlQueueKey(statement.IdentityID); byte[] value = ProtoBufUtils.Serialize(statement); RedisConnectionPool.Process(client => { client.ZAdd(key, DateTime.Now.Ticks, value); ProfileManager.PostSqlOfMessageQueueTimes(tableName, 1); }); result = true; } catch (Exception ex) { TraceLog.WriteError("Sql update queue write error:{0}\r\n{1}", ex, JsonUtils.SerializeCustom(statement)); } return(result); }
public void RedisListTest() { var watch = Stopwatch.StartNew(); var keys = new List <string> { "__List:1000:1", "__List:1000:2" }; var script = @" local result={} local index = 0 local len = table.getn(KEYS) for i=1, len do local key = KEYS[i] local values = redis.call('lrange',key,0,-1) local l = table.getn(values) result[i] = table.concat(values,',') end return result "; List <string> valueList = null; RedisConnectionPool.Process(c => { valueList = c.ExecLuaAsList(script, keys.ToArray(), new string[0]); }); Trace.WriteLine("value count:" + valueList.Count + "\r\n" + string.Join("\r\n", valueList)); WaitEnd(watch); }
public void FindRedis() { RedisConnectionPool.Process(cli => { /* * var watch = Stopwatch.StartNew(); * key = "$FrameworkUnitTest.Cache.Model.UserKeyData";//"$FrameworkUnitTest.Cache.Model.QuestProcess";// * //var keys = cli.HKeys(key); * //Trace.WriteLine("load time:" + watch.ElapsedMilliseconds + "ms." + "keys:" + keys.Length); * //var valueBytes = cli.HGet(key, keys); * //Trace.WriteLine("load time:" + watch.ElapsedMilliseconds + "ms." + "vals:" + valueBytes.Length); * byte[] suBytes = Encoding.Default.GetBytes("1380000"); * var script = @" * local userId = KEYS[2] * local key = KEYS[1] * local keyBytes = redis.call('HKeys',key) * print('keys:'.. table.getn(keyBytes)) * local valueBytes = {} * for i,k in pairs(keyBytes) do * if string.find(k, userId) then * local val = redis.call('HGet',key, k) * table.insert(valueBytes, val) * end * end * return valueBytes * "; * var keybytes = Encoding.Default.GetBytes(key); * var valueBytes = cli.Eval(script, 2, keybytes, suBytes); * Trace.WriteLine("load time:" + watch.ElapsedMilliseconds + "ms." + "vals:" + (valueBytes.Length > 0 && valueBytes[0] != null ? Encoding.Default.GetString(valueBytes[0]) + "count:" + valueBytes.Length : "count:" + valueBytes.Length)); * */ }); }
/// <summary> /// /// </summary> /// <param name="token"></param> /// <returns></returns> public static UserToken GetUserToken(string token) { UserToken userToken = null; string redisKey = string.Format("{0}@{1}", AccountServerToken, token); if (!string.IsNullOrEmpty(RedisHost) && RedisConnected) { //use redis cache. RedisConnectionPool.Process(client => { var val = client.Get(redisKey); userToken = val == null || val.Length == 0 ? new UserToken() : MathUtils.ParseJson <UserToken>(Encoding.UTF8.GetString(val)); }); return(userToken); } if (HttpContext.Current != null && HttpContext.Current.Cache != null) { userToken = (UserToken)HttpContext.Current.Cache[redisKey]; } if (Equals(userToken, null) && userTokenCache.ContainsKey(redisKey)) { userToken = (UserToken)userTokenCache[redisKey]; } return(userToken); }
public void CheckCompletedTest() { var keys = new[] { "testkey1", "testkey2" }; var vals = new[] { "1", "2" }; RedisConnectionPool.SetExpire(keys, vals, 300); RedisConnectionPool.Process(c => { var values = new List <string>(); using (var p = c.CreatePipeline()) { foreach (var key in keys) { string k = key; p.QueueCommand(cli => cli.Get <string>(k), v => { values.Add(v); }); } p.Flush(); } Trace.WriteLine(string.Join(",", values)); }); var watch = Stopwatch.StartNew(); var data = ShareCacheStruct.Get <ChildData>(1); data.Age++; bool result = CacheFactory.CheckCompleted(); Trace.WriteLine("Queue result:" + result); WaitEnd(watch); }
/// <summary> /// /// </summary> /// <param name="key"></param> /// <param name="userToken"></param> public static void SaveToken(string key, UserToken userToken) { string userKey = string.Format("{0}:{1}", AccountServerToken, userToken.UserId); string redisKey = string.Format("{0}@{1}", AccountServerToken, key); if (!string.IsNullOrEmpty(RedisHost) && RedisConnected) { try { string script = @" local redisKey = KEYS[1] local userKey = KEYS[2] local key = KEYS[3] local timeout = ARGV[1] local val = ARGV[2] local oldToken = redis.call('Get', userKey) if oldToken and oldToken ~= nil then local k = '__AccountToken@'..oldToken redis.call('del', k) end redis.call('Set', redisKey, val, 'EX', timeout) redis.call('Set', userKey, key, 'EX', timeout) return 0 "; //use redis cache. RedisConnectionPool.Process(client => { int timeout = (int)(userToken.ExpireTime - DateTime.Now).TotalSeconds; client.ExecLuaAsInt(script, new[] { redisKey, userKey, key }, new[] { timeout.ToString(), MathUtils.ToJson(userToken) }); }); } catch (Exception ex) { TraceLog.WriteError("Save token errror:{0}", ex); } return; } if (HttpContext.Current != null && HttpContext.Current.Cache != null) { var cache = HttpContext.Current.Cache; var oldToken = cache[userKey] as string; if (!string.IsNullOrEmpty(oldToken)) { cache.Remove(string.Format("{0}@{1}", AccountServerToken, oldToken)); } cache[redisKey] = userToken; cache[userKey] = key; } else { string oldToken; if (userHashCache.TryGetValue(userToken.UserId, out oldToken)) { UserToken temp; userTokenCache.TryRemove(oldToken, out temp); } userTokenCache[key] = userToken; } }
private static void OnAsyncSend(string queueKey, byte[][] keyBytes, byte[][] valueBytes) { try { RedisConnectionPool.Process(client => client.HMSet(queueKey, keyBytes, valueBytes)); } catch (Exception ex) { TraceLog.WriteError("Async send to redis's key:{0} error:{1}", queueKey, ex); } }
/// <summary> /// Send entity to db saved. /// </summary> /// <param name="entityList"></param> public static void SendToDb(params AbstractEntity[] entityList) { string key = ""; try { if (entityList == null || entityList.Length == 0) { return; } var sender = new SqlDataSender(); var groupList = entityList.GroupBy(t => t.GetIdentityId()); var sqlList = new List <KeyValuePair <string, KeyValuePair <byte[], long> > >(); foreach (var g in groupList) { var valueList = g.ToList(); foreach (var entity in valueList) { if (entity == null) { continue; } SqlStatement statement = sender.GenerateSqlQueue(entity); if (statement == null) { throw new Exception(string.Format("Generate sql of \"{0}\" entity error", entity.GetType().FullName)); } var sqlValueBytes = ProtoBufUtils.Serialize(statement); string sqlQueueKey = SqlStatementManager.GetSqlQueueKey(statement.IdentityID); sqlList.Add(new KeyValuePair <string, KeyValuePair <byte[], long> >(sqlQueueKey, new KeyValuePair <byte[], long>(sqlValueBytes, DateTime.Now.Ticks))); } } RedisConnectionPool.Process(client => { var groupSqlList = sqlList.GroupBy(t => t.Key); foreach (var g in groupSqlList) { var pairs = g.Select(t => t.Value).ToList(); client.ZAdd(g.Key, pairs); } }); } catch (Exception ex) { TraceLog.WriteError("Send To Db key:{0} error:{1}", key, ex); } }
private static void SaveTo() { try { byte[] data = ProtoBufUtils.Serialize(_globalSession); RedisConnectionPool.Process(client => client.Set(sessionRedisKey, data)); } catch (Exception er) { TraceLog.WriteError("Save session to redis faild,{0}", er); } }
/// <summary> /// 通过Redis键获取实体对象 /// </summary> /// <param name="key"></param> /// <param name="isRemove"></param> /// <param name="type"></param> /// <returns></returns> public static dynamic GetEntityFromRedis(string key, bool isRemove, Type type = null) { string typeName; string asmName; bool isEntityType; string redisKey; string entityKey = GetEntityTypeFromKey(key, out typeName, ref type, out asmName, out isEntityType, out redisKey); dynamic entity = null; RedisConnectionPool.Process(client => { if (isEntityType) { var data = client.Get <byte[]>(redisKey); if (data != null && type != null) { entity = ProtoBufUtils.Deserialize(data, type); } } else { var data = client.Get <byte[]>(redisKey); if (data != null && type != null) { var dict = (IDictionary)ProtoBufUtils.Deserialize(data, type); entity = dict[entityKey]; } } if (isRemove && entity == null && type != null) { string setId = (isEntityType ? typeName : redisKey) + ":remove"; var data = client.Get <byte[]>(setId); if (data != null) { var dict = (IDictionary)ProtoBufUtils.Deserialize(data, type); entity = dict[entityKey]; dict.Remove(entityKey); if (dict.Count > 0) { client.Set(setId, ProtoBufUtils.Serialize(dict)); } else { client.Remove(setId); } } } }); return(entity); }
/// <summary> /// /// </summary> /// <param name="token"></param> /// <param name="redisHost"></param> /// <param name="redisDb"></param> /// <returns></returns> public static UserToken GetUserToken(string token, string redisHost, int redisDb) { UserToken userToken = null; string redisKey = string.Format("{0}@{1}", AccountServerToken, token); RedisConnectionPool.Process(client => { var val = client.Get(redisKey); userToken = val == null || val.Length == 0 ? new UserToken() : MathUtils.ParseJson <UserToken>(Encoding.UTF8.GetString(val)); }, new RedisPoolSetting() { Host = redisHost, DbIndex = redisDb }); return(userToken); }
/// <summary> /// put error sql /// </summary> /// <param name="value"></param> /// <param name="key"></param> /// <returns></returns> private static bool PutError(byte[] value, string key = null) { bool result = false; try { RedisConnectionPool.Process(client => client.ZAdd(SlaveMessageQueue + (string.IsNullOrEmpty(key) ? SqlSyncErrorQueueKey : key), DateTime.Now.Ticks, value)); result = true; } catch (Exception ex) { TraceLog.WriteError("Sql error queue write error:{0}", ex); } return(result); }
/// <summary> /// put error sql /// </summary> /// <param name="value"></param> /// <returns></returns> private static bool PutError(byte[] value) { bool result = false; try { RedisConnectionPool.Process(client => client.ZAdd(SqlSyncErrorQueueKey, DateTime.Now.Ticks, value)); result = true; } catch (Exception ex) { TraceLog.WriteError("Sql error queue write error:{0}", ex); } return(result); }
/// <summary> /// 放到Redsi中 /// </summary> /// <param name="statement"></param> public static bool Put(SqlStatement statement) { bool result = false; try { string key = GetSqlQueueKey(statement.IdentityID); byte[] value = ProtoBufUtils.Serialize(statement); RedisConnectionPool.Process(client => client.ZAdd(key, DateTime.Now.Ticks, value)); result = true; } catch (Exception ex) { TraceLog.WriteError("Sql update queue write error:{0}\r\n{1}", ex, JsonUtils.SerializeCustom(statement)); } return(result); }
public void RedisTestConnect() { var date = DateTime.Now; var tasks = new Task[1000]; for (int i = 0; i < tasks.Count(); i++) { tasks[i] = Task.Factory.StartNew(() => { while (date.AddMinutes(1) > DateTime.Now) { RedisConnectionPool.Process(c => c.HLen("$GameServer.Model.GameUser")); Thread.Sleep(100); } }); } Task.WaitAll(tasks); }
/// <summary> /// /// </summary> /// <param name="key"></param> /// <param name="userToken"></param> public static void SaveToken(string key, UserToken userToken) { string userKey = string.Format("{0}:{1}", AccountServerToken, userToken.UserId); string redisKey = string.Format("{0}@{1}", AccountServerToken, key); if (!string.IsNullOrEmpty(RedisHost) && RedisConnected) { //use redis cache. RedisConnectionPool.Process(client => { string oldToken = client.Get <string>(userKey); if (!string.IsNullOrEmpty(oldToken)) { client.Remove(string.Format("{0}@{1}", AccountServerToken, oldToken)); } client.Set(redisKey, userToken, userToken.ExpireTime); client.Set(userKey, key, userToken.ExpireTime); }); return; } if (HttpContext.Current != null && HttpContext.Current.Cache != null) { var cache = HttpContext.Current.Cache; var oldToken = cache[userKey] as string; if (!string.IsNullOrEmpty(oldToken)) { cache.Remove(string.Format("{0}@{1}", AccountServerToken, oldToken)); } cache[redisKey] = userToken; cache[userKey] = key; } else { string oldToken; if (userHashCache.TryGetValue(userToken.UserId, out oldToken)) { UserToken temp; userTokenCache.TryRemove(oldToken, out temp); } userTokenCache[key] = userToken; } }
/// <summary> /// Send to queue pool /// </summary> /// <param name="entityList"></param> public static void Send(AbstractEntity[] entityList) { string key = ""; try { if (entityList == null || entityList.Length == 0) { return; } var groupList = entityList.GroupBy(t => t.GetIdentityId()); foreach (var g in groupList) { var valueList = g.ToList(); byte[][] keyBytes = new byte[valueList.Count][]; byte[][] valueBytes = new byte[valueList.Count][]; string queueKey = GetRedisSyncQueueKey(g.Key); byte[] idBytes = BufferUtils.GetBytes(g.Key); int index = 0; foreach (var entity in valueList) { key = string.Format("{0}_{1}", entity.GetType().FullName, entity.GetKeyCode()); keyBytes[index] = RedisConnectionPool.ToByteKey(key); byte[] stateBytes = BufferUtils.GetBytes(entity.IsDelete ? 1 : 0); valueBytes[index] = BufferUtils.MergeBytes( BufferUtils.GetBytes(idBytes.Length + stateBytes.Length), idBytes, stateBytes, ProtoBufUtils.Serialize(entity)); index++; } RedisConnectionPool.Process(client => client.HMSet(queueKey, keyBytes, valueBytes)); } } catch (Exception ex) { TraceLog.WriteError("Post changed key:{0} error:{1}", key, ex); } }
/// <summary> /// Process synchronized wait queue of Sql /// </summary> /// <param name="waitQueueKey"></param> /// <param name="keys"></param> /// <param name="values"></param> private static void DoProcessSqlWaitSyncQueue(string waitQueueKey, byte[][] keys, byte[][] values) { try { var sqlSender = new SqlDataSender(); RedisConnectionPool.Process(client => { var sqlList = GenerateSqlFrom(sqlSender, client, keys, values); var groupSqlList = sqlList.GroupBy(t => t.Key); foreach (var g in groupSqlList) { var pairs = g.Select(t => t.Value).ToList(); client.ZAdd(g.Key, pairs); } }); } catch (Exception ex) { TraceLog.WriteError("DoProcessSqlWaitSyncQueue error:{0}", ex); } }
/// <summary> /// /// </summary> /// <param name="key"></param> /// <returns></returns> public static UserToken GetUserToken(string key) { UserToken userToken = null; if (!string.IsNullOrEmpty(RedisHost) && RedisConnected) { //use redis cache. RedisConnectionPool.Process(client => { userToken = client.Get <UserToken>(string.Format("{0}@{1}", AccountServerToken, key)); }); return(userToken); } if (HttpContext.Current != null && HttpContext.Current.Cache != null) { userToken = (UserToken)HttpContext.Current.Cache[key]; } if (Equals(userToken, null) && userTokenCache.ContainsKey(key)) { userToken = (UserToken)userTokenCache[key]; } return(userToken); }
/// <summary> /// Process synchronized queue of redis /// </summary> /// <param name="sysnWorkingQueueKey"></param> /// <param name="keys"></param> /// <param name="values"></param> private static bool ProcessRedisSyncQueue(string sysnWorkingQueueKey, byte[][] keys, byte[][] values) { bool result = false; try { var redisSyncErrorQueue = new List <byte[][]>(); var entityList = new List <RedisEntityItem>(); var entityRemList = new List <RedisEntityItem>(); var mutilKeyMapList = new List <RedisEntityItem>(); var mutilKeyMapRemList = new List <RedisEntityItem>(); var sqlWaitSyncQueue = new List <KeyValuePair <string, byte[][]> >(); for (int i = 0; i < keys.Length; i++) { byte[] keyBytes = keys[i]; byte[] valueBytes = values[i]; try { string[] queueKey = RedisConnectionPool.ToStringKey(keyBytes).Split('_'); string entityTypeName = RedisConnectionPool.DecodeTypeName(queueKey[0]); string entityParentKey = RedisConnectionPool.GetRedisEntityKeyName(queueKey[0]); byte[] entityKeyBytes = RedisConnectionPool.ToByteKey(queueKey[1]); bool hasMutilKey = false; bool isStoreInDb = true; SchemaTable schema; if (EntitySchemaSet.TryGet(entityTypeName, out schema)) { hasMutilKey = RedisConnectionPool.CurrRedisInfo.ClientVersion >= RedisStorageVersion.HashMutilKeyMap && schema.EntityType.IsSubclassOf(typeof(BaseEntity)) && schema.Keys.Length > 1; isStoreInDb = schema.StorageType.HasFlag(StorageType.WriteOnlyDB) || schema.StorageType.HasFlag(StorageType.ReadWriteDB); } byte[] headBytes; byte[] entityValBytes; int state; int identity; DecodeValueBytes(valueBytes, out headBytes, out entityValBytes, out state, out identity); var entityItem = new RedisEntityItem() { HashId = entityParentKey, UserId = identity, KeyBytes = entityKeyBytes, ValueBytes = entityValBytes, State = state, HasMutilKey = hasMutilKey }; if (entityItem.State == 1) { entityRemList.Add(entityItem); if (hasMutilKey) { mutilKeyMapRemList.Add(entityItem); } } else { entityList.Add(entityItem); if (hasMutilKey) { mutilKeyMapList.Add(entityItem); } } if (_enableWriteToDb && isStoreInDb) { //增加到Sql等待队列 string sqlWaitQueueKey = GetSqlWaitSyncQueueKey(identity); sqlWaitSyncQueue.Add(new KeyValuePair <string, byte[][]>(sqlWaitQueueKey, new[] { keyBytes, headBytes })); } } catch { redisSyncErrorQueue.Add(new[] { keyBytes, valueBytes }); } } var redisErrorKeys = redisSyncErrorQueue.Select(p => p[0]).ToArray(); var redisErrorValues = redisSyncErrorQueue.Select(p => p[1]).ToArray(); var sqlWaitGroups = sqlWaitSyncQueue.GroupBy(p => p.Key); var setGroups = entityList.GroupBy(p => p.HashId); var removeGroups = entityRemList.GroupBy(p => p.HashId); var mutilKeyMapGroups = mutilKeyMapList.GroupBy(p => p.HashId); var mutilKeyMapRemGroups = mutilKeyMapRemList.GroupBy(p => p.HashId); RedisConnectionPool.ProcessPipeline(pipeline => { bool hasPost = false; foreach (var g in setGroups) { var entityKeys = g.Select(p => p.KeyBytes).ToArray(); var entityValues = g.Select(p => p.ValueBytes).ToArray(); pipeline.QueueCommand(client => ((RedisClient)client).HMSet(g.Key, entityKeys, entityValues)); hasPost = true; } foreach (var g in removeGroups) { var keybytes = g.Select(p => p.KeyBytes).ToArray(); pipeline.QueueCommand(client => ((RedisClient)client).HDel(g.Key, keybytes)); hasPost = true; } foreach (var g in mutilKeyMapGroups) { string hashId = g.Key; var subGroup = g.GroupBy(t => t.UserId); foreach (var @group in subGroup) { string firstKey = AbstractEntity.EncodeKeyCode(@group.Key.ToString()); var keybytes = @group.Select(p => p.KeyBytes).ToArray(); pipeline.QueueCommand(client => RedisConnectionPool.SetMutilKeyMap((RedisClient)client, hashId, firstKey, keybytes)); hasPost = true; } } foreach (var g in mutilKeyMapRemGroups) { string hashId = g.Key; var subGroup = g.GroupBy(t => t.UserId); foreach (var @group in subGroup) { string firstKey = AbstractEntity.EncodeKeyCode(@group.Key.ToString()); var keybytes = @group.Select(p => p.KeyBytes).ToArray(); pipeline.QueueCommand(client => RedisConnectionPool.RemoveMutilKeyMap((RedisClient)client, hashId, firstKey, keybytes)); hasPost = true; } } if (redisErrorKeys.Length > 0) { pipeline.QueueCommand(client => ((RedisClient)client).HMSet(RedisSyncErrorQueueKey, redisErrorKeys, redisErrorValues)); hasPost = true; } foreach (var g in sqlWaitGroups) { var sqlWaitKeys = g.Select(p => p.Value[0]).ToArray(); var sqlWaitValues = g.Select(p => p.Value[1]).ToArray(); pipeline.QueueCommand(client => ((RedisClient)client).HMSet(g.Key, sqlWaitKeys, sqlWaitValues)); hasPost = true; } if (hasPost) { pipeline.Flush(); } result = redisErrorKeys.Length == 0; }); } catch (Exception ex) { TraceLog.WriteError("DoProcessRedisSyncQueue error:{0}", ex); try { RedisConnectionPool.Process(client => client.HMSet(RedisSyncErrorQueueKey, keys, values)); } catch (Exception er) { TraceLog.WriteError("Put RedisSyncErrorQueue error:{0}", er); } } return(result); }
/// <summary> /// Check to be synchronized wait queue of Sql /// </summary> /// <param name="state"></param> private static void OnCheckSqlWaitSyncQueue(object state) { int identity = (int)state; if (Interlocked.CompareExchange(ref _isSqlWaitSyncWorking[identity], 1, 0) == 0) { try { string queueKey = GetSqlWaitSyncQueueKey(identity); string workingKey = queueKey + "_temp"; byte[][] keys = null; byte[][] values = null; RedisConnectionPool.ProcessReadOnly(client => { bool hasWorkingQueue = client.HLen(workingKey) > 0; bool hasNewWorkingQueue = client.HLen(queueKey) > 0; if (!hasWorkingQueue && !hasNewWorkingQueue) { return; } if (!hasWorkingQueue) { try { client.Rename(queueKey, workingKey); } catch { } } var keyValuePairs = client.HGetAll(workingKey); if (keyValuePairs != null && keyValuePairs.Length > 0) { keys = keyValuePairs.Where((buffs, index) => index % 2 == 0).ToArray(); values = keyValuePairs.Where((buffs, index) => index % 2 == 1).ToArray(); client.Remove(workingKey); } }); if (keys != null && keys.Length > 0) { try { int pos = 0; byte[][] subKeys, subValues; while (true) { int count = pos + sqlWaitPackSize < keys.Length ? sqlWaitPackSize : keys.Length - pos; if (count <= 0) { break; } subKeys = new byte[count][]; subValues = new byte[count][]; Array.Copy(keys, pos, subKeys, 0, subKeys.Length); Array.Copy(values, pos, subValues, 0, subValues.Length); _threadPools.QueueWorkItem(DoProcessSqlWaitSyncQueue, workingKey, subKeys, subValues); pos += sqlWaitPackSize; } } catch (Exception er) { TraceLog.WriteError("OnCheckSqlWaitSyncQueue error:{0}", er); RedisConnectionPool.Process(client => client.HMSet(SqlSyncWaitErrirQueueKey, keys, values)); } } } catch (Exception ex) { TraceLog.WriteError("OnCheckSqlWaitSyncQueue error:{0}", ex); } finally { Interlocked.Exchange(ref _isSqlWaitSyncWorking[identity], 0); } } }
/// <summary> /// Process synchronized queue of redis /// </summary> /// <param name="sysnWorkingQueueKey"></param> /// <param name="keys"></param> /// <param name="values"></param> private static void DoProcessRedisSyncQueue(string sysnWorkingQueueKey, byte[][] keys, byte[][] values) { try { var redisSyncErrorQueue = new List <byte[][]>(); var setList = new List <KeyValuePair <string, byte[][]> >(); var removeList = new List <KeyValuePair <string, byte[]> >(); var sqlWaitSyncQueue = new List <KeyValuePair <string, byte[][]> >(); for (int i = 0; i < keys.Length; i++) { byte[] keyBytes = keys[i]; byte[] valueBytes = values[i]; try { string[] queueKey = RedisConnectionPool.ToStringKey(keyBytes).Split('_'); string entityTypeName = RedisConnectionPool.DecodeTypeName(queueKey[0]); string entityParentKey = RedisConnectionPool.GetRedisEntityKeyName(queueKey[0]); byte[] entityKeyBytes = RedisConnectionPool.ToByteKey(queueKey[1]); byte[] headBytes; byte[] entityValBytes; int state; int identity; DecodeValueBytes(valueBytes, out headBytes, out entityValBytes, out state, out identity); if (state == 1) { removeList.Add(new KeyValuePair <string, byte[]>(entityParentKey, entityKeyBytes)); } else { setList.Add(new KeyValuePair <string, byte[][]>(entityParentKey, new[] { entityKeyBytes, entityValBytes })); } bool isStoreInDb = true; SchemaTable schema; if (EntitySchemaSet.TryGet(entityTypeName, out schema)) { isStoreInDb = schema.StorageType.HasFlag(StorageType.WriteOnlyDB) || schema.StorageType.HasFlag(StorageType.ReadWriteDB); } if (_enableWriteToDb && isStoreInDb) { //增加到Sql等待队列 string sqlWaitQueueKey = GetSqlWaitSyncQueueKey(identity); sqlWaitSyncQueue.Add(new KeyValuePair <string, byte[][]>(sqlWaitQueueKey, new[] { keyBytes, headBytes })); } } catch { redisSyncErrorQueue.Add(new[] { keyBytes, valueBytes }); } } var redisErrorKeys = redisSyncErrorQueue.Select(p => p[0]).ToArray(); var redisErrorValues = redisSyncErrorQueue.Select(p => p[1]).ToArray(); var sqlWaitGroups = sqlWaitSyncQueue.GroupBy(p => p.Key); var setGroups = setList.GroupBy(p => p.Key); var removeGroups = removeList.GroupBy(p => p.Key); RedisConnectionPool.Process(client => { foreach (var g in setGroups) { var entityKeys = g.Select(p => p.Value[0]).ToArray(); var entityValues = g.Select(p => p.Value[1]).ToArray(); client.HMSet(g.Key, entityKeys, entityValues); } foreach (var g in removeGroups) { var keybytes = g.Select(p => p.Value).ToArray(); client.HDel(g.Key, keybytes); } if (redisErrorKeys.Length > 0) { client.HMSet(RedisSyncErrorQueueKey, redisErrorKeys, redisErrorValues); } foreach (var g in sqlWaitGroups) { var sqlWaitKeys = g.Select(p => p.Value[0]).ToArray(); var sqlWaitValues = g.Select(p => p.Value[1]).ToArray(); client.HMSet(g.Key, sqlWaitKeys, sqlWaitValues); } }); } catch (Exception ex) { TraceLog.WriteError("DoProcessRedisSyncQueue error:{0}", ex); try { RedisConnectionPool.Process(client => client.HMSet(RedisSyncErrorQueueKey, keys, values)); } catch (Exception er) { TraceLog.WriteError("Put RedisSyncErrorQueue error:{0}", er); } } }
private static void OnEntitySyncQueue(object state) { if (Interlocked.CompareExchange(ref _entityQueueRunning, 1, 0) == 0) { try { var tempRemove = Interlocked.Exchange(ref _entityRemoteSet, new HashSet <string>()); var temp = Interlocked.Exchange(ref _entitySet, new HashSet <string>()); if (temp.Count == 0 || _queueWatchTimers == null) { return; } TraceLog.WriteWarn("OnEntitySyncQueue execute count:{0}, success:{1}/total {2}, fail:{3} start...", temp.Count, ExecuteSuccessCount, SendWaitCount, ExecuteFailCount); RedisConnectionPool.Process(client => { while (temp.Count > 0) { var dumpSet = temp.Take(ExecuteCount).ToArray(); var pipeline = client.CreatePipeline(); try { bool hasPost = false; foreach (var key in dumpSet) { var keyValues = key.Split('_', '|'); if (keyValues.Length != 3) { TraceLog.WriteWarn("OnEntitySyncQueue:{0}", key); ExecuteFailCount++; continue; } AbstractEntity entity = CacheFactory.GetPersonalEntity(key) as AbstractEntity; int id = keyValues[1].ToInt(); string keyCode = keyValues[2]; string redisKey = string.Format("{0}_{1}", keyValues[0], keyCode); string hashId = GetRedisSyncQueueKey(id); byte[] idBytes = BufferUtils.GetBytes(id); var keyBytes = RedisConnectionPool.ToByteKey(redisKey); bool isDelete; byte[] entityBytes; if (entity != null) { isDelete = entity.IsDelete; entityBytes = _serializer.Serialize(entity); //modify resean: set unchange status. entity.Reset(); } else if (tempRemove.Contains(key)) { entityBytes = new byte[0]; isDelete = true; } else { ExecuteFailCount++; TraceLog.WriteError("EntitySync queue key {0} faild object is null.", key); continue; } byte[] stateBytes = BufferUtils.GetBytes(isDelete ? 1 : 0); byte[] values = BufferUtils.MergeBytes(BufferUtils.GetBytes(idBytes.Length + stateBytes.Length), idBytes, stateBytes, entityBytes); pipeline.QueueCommand(c => { ((RedisClient)c).HSet(hashId, keyBytes, values); }); hasPost = true; ExecuteSuccessCount++; } if (hasPost) { pipeline.Flush(); } } finally { pipeline.Dispose(); } try { foreach (var key in dumpSet) { temp.Remove(key); } } catch (Exception) { } } }); } catch (Exception ex) { TraceLog.WriteError("EntitySync queue {0}", ex); } finally { Interlocked.Exchange(ref _entityQueueRunning, 0); } } }
/// <summary> /// 从历史库中加载数据 /// </summary> /// <typeparam name="T"></typeparam> /// <param name="redisKey"></param> /// <param name="dataList"></param> /// <returns></returns> public bool TryLoadHistory <T>(string redisKey, out List <T> dataList) where T : ISqlEntity { string[] entityAndKeys = (redisKey ?? "").Split('_'); var entityKey = entityAndKeys.Length > 1 ? entityAndKeys[1] : null; string entityNameKey = RedisConnectionPool.GetRedisEntityKeyName(entityAndKeys[0]); bool result = false; dataList = null; SchemaTable schemaTable; if (EntitySchemaSet.TryGet <EntityHistory>(out schemaTable)) { try { var provider = DbConnectionProvider.CreateDbProvider(schemaTable); if (provider == null) { //DB is optional and can no DB configuration dataList = new List <T>(); return(true); } TransReceiveParam receiveParam = new TransReceiveParam(entityNameKey); receiveParam.Schema = schemaTable; int maxCount = receiveParam.Schema.Capacity; var filter = new DbDataFilter(maxCount); string key = schemaTable.Keys[0]; var entitySchema = EntitySchemaSet.Get(entityAndKeys[0]); if (entitySchema != null && entitySchema.Keys.Length == 1) { filter.Condition = provider.FormatFilterParam(key); filter.Parameters.Add(key, string.Format("{0}_{1}", entityNameKey, entityKey)); } else { filter.Condition = provider.FormatFilterParam(key, "LIKE"); filter.Parameters.Add(key, string.Format("{0}_%{1}%", entityNameKey, entityKey)); } receiveParam.DbFilter = filter; List <EntityHistory> historyList; if (_dbTransponder.TryReceiveData(receiveParam, out historyList)) { if (historyList.Count == 0) { dataList = new List <T>(); return(true); } dataList = new List <T>(); var keyBytes = new byte[historyList.Count][]; var valueBytes = new byte[historyList.Count][]; for (int i = 0; i < keyBytes.Length; i++) { var entityHistory = historyList[i]; keyBytes[i] = RedisConnectionPool.ToByteKey(entityHistory.Key.Split('_')[1]); valueBytes[i] = entityHistory.Value; dataList.Add((T)_serializer.Deserialize(entityHistory.Value, typeof(T))); } //从DB备份中恢复到Redis, 多个Key时也要更新 var entitys = dataList.ToArray(); RedisConnectionPool.Process(client => { client.HMSet(entityNameKey, keyBytes, valueBytes); if (entitySchema.Keys.Length > 1) { RedisConnectionPool.UpdateFromMutilKeyMap <T>(client, entityKey, entitys); } }); result = true; } } catch (Exception ex) { TraceLog.WriteError("Try load Redis's history key:{0}\r\nerror:{1}", entityNameKey, ex); } } else { dataList = new List <T>(); result = true; } return(result); }
/// <summary> /// 通过Redis键获取实体对象 /// </summary> /// <param name="key"></param> /// <param name="isRemove"></param> /// <param name="type"></param> /// <param name="serializer"></param> /// <returns></returns> public static dynamic GetEntityFromRedis(string key, bool isRemove, Type type, ICacheSerializer serializer) { string typeName; string asmName; bool isEntityType; string redisKey; string entityKey = GetEntityTypeFromKey(key, out typeName, ref type, out asmName, out isEntityType, out redisKey); dynamic entity = null; RedisConnectionPool.Process(client => { if (isEntityType) { var data = client.Get <byte[]>(redisKey); if (data != null && type != null) { entity = serializer.Deserialize(data, type); } } else { var data = client.Get <byte[]>(redisKey); if (data != null && type != null) { var dict = (IDictionary)serializer.Deserialize(data, type); entity = dict[entityKey]; } } if (entity == null) { //新版本Hash格式 var data = client.HGet(typeName, RedisConnectionPool.ToByteKey(entityKey)); if (data != null && type != null) { entity = serializer.Deserialize(data, type); } } if (isRemove && entity == null && type != null) { //临时队列删除Entity string setId = (isEntityType ? RedisConnectionPool.EncodeTypeName(typeName) : redisKey) + ":remove"; IDictionary dict = null; RedisConnectionPool.ProcessTrans(client, new string[] { setId }, () => { var data = client.Get <byte[]>(setId); if (data == null) { return(false); } dict = (IDictionary)serializer.Deserialize(data, type); entity = dict[entityKey]; dict.Remove(entityKey); return(true); }, trans => { if (dict != null && dict.Count > 0) { trans.QueueCommand(c => c.Set(setId, serializer.Serialize(dict))); } else { trans.QueueCommand(c => c.Remove(setId)); } }, null); } }); return(entity); }
public bool TryLoadHistory <T>(string redisKey, out List <T> dataList) { redisKey = RedisConnectionPool.GetRedisEntityKeyName(redisKey); bool result = false; dataList = null; SchemaTable schemaTable; if (EntitySchemaSet.TryGet <EntityHistory>(out schemaTable)) { try { var provider = DbConnectionProvider.CreateDbProvider(schemaTable); if (provider == null) { //DB is optional and can no DB configuration dataList = new List <T>(); return(true); } TransReceiveParam receiveParam = new TransReceiveParam(redisKey); receiveParam.Schema = schemaTable; int maxCount = receiveParam.Schema.Capacity; var filter = new DbDataFilter(maxCount); string key = schemaTable.Keys[0]; filter.Condition = provider.FormatFilterParam(key); filter.Parameters.Add(key, redisKey); receiveParam.DbFilter = filter; receiveParam.Capacity = maxCount; List <EntityHistory> historyList; if (_dbTransponder.TryReceiveData(receiveParam, out historyList)) { EntityHistory history = historyList.Count > 0 ? historyList[0] : null; if (history != null && history.Value != null && history.Value.Length > 0) { byte[][] bufferBytes = ProtoBufUtils.Deserialize <byte[][]>(history.Value); byte[][] keys = bufferBytes.Where((b, index) => index % 2 == 0).ToArray(); byte[][] values = bufferBytes.Where((b, index) => index % 2 == 1).ToArray(); RedisConnectionPool.Process(client => client.HMSet(redisKey, keys, values)); dataList = values.Select(value => ProtoBufUtils.Deserialize <T>(value)).ToList(); result = true; } else { dataList = new List <T>(); result = true; } } } catch (Exception ex) { TraceLog.WriteError("Try load Redis's history key:{0}\r\nerror:{1}", redisKey, ex); } } else { dataList = new List <T>(); result = true; } return(result); }