/// <summary> /// Send entity to db saved. /// </summary> /// <param name="postColumnFunc"></param> /// <param name="getPropertyFunc"></param> /// <param name="entityList"></param> public static void SendToDb <T>(EntityPropertyGetFunc <T> getPropertyFunc, EnttiyPostColumnFunc <T> postColumnFunc, params T[] entityList) where T : ISqlEntity { string key = ""; try { if (entityList == null || entityList.Length == 0) { return; } var sender = new SqlDataSender(false); var groupList = entityList.GroupBy(t => t.GetMessageQueueId()); var sqlList = new List <KeyValuePair <string, KeyValuePair <byte[], long> > >(); foreach (var g in groupList) { key = g.Key.ToString(); var valueList = g.ToList(); foreach (var entity in valueList) { if (entity == null) { continue; } SqlStatement statement = sender.GenerateSqlQueue <T>(entity, getPropertyFunc, postColumnFunc); if (statement == null) { throw new Exception(string.Format("Generate sql of \"{0}\" entity error", entity.GetType().FullName)); } var sqlValueBytes = ProtoBufUtils.Serialize(statement); string sqlQueueKey = SqlStatementManager.GetSqlQueueKey(statement.IdentityID); sqlList.Add(new KeyValuePair <string, KeyValuePair <byte[], long> >(sqlQueueKey, new KeyValuePair <byte[], long>(sqlValueBytes, DateTime.Now.Ticks))); } } RedisConnectionPool.ProcessPipeline(p => { bool hasPost = false; var groupSqlList = sqlList.GroupBy(t => t.Key); foreach (var g in groupSqlList) { var pairs = g.Select(t => t.Value).ToList(); p.QueueCommand(client => ((RedisClient)client).ZAdd(g.Key, pairs)); hasPost = true; } if (hasPost) { p.Flush(); } }); } catch (Exception ex) { TraceLog.WriteError("Send To Db key:{0} error:{1}", key, ex); } }
/// <summary> /// Process synchronized wait queue of Sql /// </summary> /// <param name="waitQueueKey"></param> /// <param name="keys"></param> /// <param name="values"></param> private static void DoProcessSqlWaitSyncQueue(string waitQueueKey, byte[][] keys, byte[][] values) { try { var sqlSender = new SqlDataSender(false); IEnumerable <KeyValuePair <string, KeyValuePair <byte[], long> > > sqlList = null; RedisConnectionPool.ProcessPipeline(client => { sqlList = GenerateSqlFrom(sqlSender, client, keys, values); }, p => { if (sqlList == null) { return(0); } var keyValuePairs = sqlList.ToList(); var groupSqlList = keyValuePairs.GroupBy(t => t.Key); int sqlCount = keyValuePairs.Count(); bool hasPost = false; long result = 0; foreach (var g in groupSqlList) { var pairs = g.Select(t => t.Value).ToList(); p.QueueCommand(c => ((RedisClient)c).ZAdd(g.Key, pairs), r => {//onsuccess, 已经存在的返回值为0 result += r; ProfileManager.PostSqlOfMessageQueueTimes(g.Key, sqlCount); }); hasPost = true; } if (hasPost) { p.Flush(); } return(result); }); } catch (Exception ex) { TraceLog.WriteError("DoProcessSqlWaitSyncQueue error:{0}", ex); } }
public void RedisPirpe() { RedisConnectionPool.ProcessPipeline(p => { p.QueueCommand(c => c.Set("_test", "1")); p.Flush(); }); RedisConnectionPool.ProcessPipeline(client => { Assert.AreEqual(client.Get <string>("_test"), "1"); }, p => { long result = 0; p.QueueCommand(c => { result = c.Set("_test", "2") ? 1 : 0; }); p.Flush(); return(result); }); }
/// <summary> /// Process synchronized queue of redis /// </summary> /// <param name="sysnWorkingQueueKey"></param> /// <param name="keys"></param> /// <param name="values"></param> private static bool ProcessRedisSyncQueue(string sysnWorkingQueueKey, byte[][] keys, byte[][] values) { bool result = false; try { var redisSyncErrorQueue = new List <byte[][]>(); var entityList = new List <RedisEntityItem>(); var entityRemList = new List <RedisEntityItem>(); var mutilKeyMapList = new List <RedisEntityItem>(); var mutilKeyMapRemList = new List <RedisEntityItem>(); var sqlWaitSyncQueue = new List <KeyValuePair <string, byte[][]> >(); for (int i = 0; i < keys.Length; i++) { byte[] keyBytes = keys[i]; byte[] valueBytes = values[i]; try { string[] queueKey = RedisConnectionPool.ToStringKey(keyBytes).Split('_'); string entityTypeName = RedisConnectionPool.DecodeTypeName(queueKey[0]); string entityParentKey = RedisConnectionPool.GetRedisEntityKeyName(queueKey[0]); byte[] entityKeyBytes = RedisConnectionPool.ToByteKey(queueKey[1]); bool hasMutilKey = false; bool isStoreInDb = true; SchemaTable schema; if (EntitySchemaSet.TryGet(entityTypeName, out schema)) { hasMutilKey = RedisConnectionPool.CurrRedisInfo.ClientVersion >= RedisStorageVersion.HashMutilKeyMap && schema.EntityType.IsSubclassOf(typeof(BaseEntity)) && schema.Keys.Length > 1; isStoreInDb = schema.StorageType.HasFlag(StorageType.WriteOnlyDB) || schema.StorageType.HasFlag(StorageType.ReadWriteDB); } byte[] headBytes; byte[] entityValBytes; int state; int identity; DecodeValueBytes(valueBytes, out headBytes, out entityValBytes, out state, out identity); var entityItem = new RedisEntityItem() { HashId = entityParentKey, UserId = identity, KeyBytes = entityKeyBytes, ValueBytes = entityValBytes, State = state, HasMutilKey = hasMutilKey }; if (entityItem.State == 1) { entityRemList.Add(entityItem); if (hasMutilKey) { mutilKeyMapRemList.Add(entityItem); } } else { entityList.Add(entityItem); if (hasMutilKey) { mutilKeyMapList.Add(entityItem); } } if (_enableWriteToDb && isStoreInDb) { //增加到Sql等待队列 string sqlWaitQueueKey = GetSqlWaitSyncQueueKey(identity); sqlWaitSyncQueue.Add(new KeyValuePair <string, byte[][]>(sqlWaitQueueKey, new[] { keyBytes, headBytes })); } } catch { redisSyncErrorQueue.Add(new[] { keyBytes, valueBytes }); } } var redisErrorKeys = redisSyncErrorQueue.Select(p => p[0]).ToArray(); var redisErrorValues = redisSyncErrorQueue.Select(p => p[1]).ToArray(); var sqlWaitGroups = sqlWaitSyncQueue.GroupBy(p => p.Key); var setGroups = entityList.GroupBy(p => p.HashId); var removeGroups = entityRemList.GroupBy(p => p.HashId); var mutilKeyMapGroups = mutilKeyMapList.GroupBy(p => p.HashId); var mutilKeyMapRemGroups = mutilKeyMapRemList.GroupBy(p => p.HashId); RedisConnectionPool.ProcessPipeline(pipeline => { bool hasPost = false; foreach (var g in setGroups) { var entityKeys = g.Select(p => p.KeyBytes).ToArray(); var entityValues = g.Select(p => p.ValueBytes).ToArray(); pipeline.QueueCommand(client => ((RedisClient)client).HMSet(g.Key, entityKeys, entityValues)); hasPost = true; } foreach (var g in removeGroups) { var keybytes = g.Select(p => p.KeyBytes).ToArray(); pipeline.QueueCommand(client => ((RedisClient)client).HDel(g.Key, keybytes)); hasPost = true; } foreach (var g in mutilKeyMapGroups) { string hashId = g.Key; var subGroup = g.GroupBy(t => t.UserId); foreach (var @group in subGroup) { string firstKey = AbstractEntity.EncodeKeyCode(@group.Key.ToString()); var keybytes = @group.Select(p => p.KeyBytes).ToArray(); pipeline.QueueCommand(client => RedisConnectionPool.SetMutilKeyMap((RedisClient)client, hashId, firstKey, keybytes)); hasPost = true; } } foreach (var g in mutilKeyMapRemGroups) { string hashId = g.Key; var subGroup = g.GroupBy(t => t.UserId); foreach (var @group in subGroup) { string firstKey = AbstractEntity.EncodeKeyCode(@group.Key.ToString()); var keybytes = @group.Select(p => p.KeyBytes).ToArray(); pipeline.QueueCommand(client => RedisConnectionPool.RemoveMutilKeyMap((RedisClient)client, hashId, firstKey, keybytes)); hasPost = true; } } if (redisErrorKeys.Length > 0) { pipeline.QueueCommand(client => ((RedisClient)client).HMSet(RedisSyncErrorQueueKey, redisErrorKeys, redisErrorValues)); hasPost = true; } foreach (var g in sqlWaitGroups) { var sqlWaitKeys = g.Select(p => p.Value[0]).ToArray(); var sqlWaitValues = g.Select(p => p.Value[1]).ToArray(); pipeline.QueueCommand(client => ((RedisClient)client).HMSet(g.Key, sqlWaitKeys, sqlWaitValues)); hasPost = true; } if (hasPost) { pipeline.Flush(); } result = redisErrorKeys.Length == 0; }); } catch (Exception ex) { TraceLog.WriteError("DoProcessRedisSyncQueue error:{0}", ex); try { RedisConnectionPool.Process(client => client.HMSet(RedisSyncErrorQueueKey, keys, values)); } catch (Exception er) { TraceLog.WriteError("Put RedisSyncErrorQueue error:{0}", er); } } return(result); }
private static void OnEntitySyncQueue(object state) { if (Interlocked.CompareExchange(ref _entityQueueRunning, 1, 0) == 0) { try { var tempRemove = Interlocked.Exchange(ref _entityRemoteSet, new HashSet <string>()); var temp = Interlocked.Exchange(ref _entitySet, new HashSet <string>()); if (temp.Count == 0 || _queueWatchTimers == null) { return; } TraceLog.WriteWarn("OnEntitySyncQueue execute count:{0}, success:{1}/total {2}, fail:{3} start...", temp.Count, ExecuteSuccessCount, SendWaitCount, ExecuteFailCount); RedisConnectionPool.ProcessPipeline(pipeline => { bool hasPost = false; foreach (var key in temp) { var keyValues = key.Split('_', '|'); if (keyValues.Length != 3) { TraceLog.WriteWarn("OnEntitySyncQueue:{0}", key); ExecuteFailCount++; continue; } AbstractEntity entity = CacheFactory.GetPersonalEntity(key) as AbstractEntity; int id = AbstractEntity.DecodeKeyCode(keyValues[1]).ToInt(); string keyCode = keyValues[2]; string redisKey = string.Format("{0}_{1}", keyValues[0], keyCode); string hashId = GetRedisSyncQueueKey(id); byte[] idBytes = BufferUtils.GetBytes(id); var keyBytes = RedisConnectionPool.ToByteKey(redisKey); bool isDelete; byte[] entityBytes; if (entity != null) { isDelete = entity.IsDelete; entityBytes = _serializer.Serialize(entity); //modify resean: set unchange status. entity.Reset(); } else if (tempRemove.Contains(key)) { entityBytes = new byte[0]; isDelete = true; } else { ExecuteFailCount++; TraceLog.WriteError("EntitySync queue key {0} faild object is null.", key); continue; } byte[] stateBytes = BufferUtils.GetBytes(isDelete ? 1 : 0); byte[] values = BufferUtils.MergeBytes(BufferUtils.GetBytes(idBytes.Length + stateBytes.Length), idBytes, stateBytes, entityBytes); pipeline.QueueCommand(c => { ((RedisClient)c).HSet(hashId, keyBytes, values); }); hasPost = true; ExecuteSuccessCount++; } if (hasPost) { pipeline.Flush(); } }); } catch (Exception ex) { TraceLog.WriteError("EntitySync queue {0}", ex); } finally { Interlocked.Exchange(ref _entityQueueRunning, 0); } } }
/// <summary> /// Send entity to db saved. /// </summary> /// <param name="postColumnFunc"></param> /// <param name="getPropertyFunc"></param> /// <param name="entityList"></param> public static void SendToDb <T>(string connectKey, EntityPropertyGetFunc <T> getPropertyFunc, EnttiyPostColumnFunc <T> postColumnFunc, params T[] entityList) where T : ISqlEntity { string key = ""; try { if (entityList == null || entityList.Length == 0) { return; } var sender = new SqlDataSender(false, connectKey); var groupList = entityList.GroupBy(t => t.GetMessageQueueId()); var sqlList = new List <KeyValuePair <string, byte[]> >(); foreach (var g in groupList) { key = g.Key.ToString(); var valueList = g.ToList(); foreach (var entity in valueList) { if (entity == null) { continue; } SqlStatement statement = sender.GenerateSqlQueue <T>(entity, getPropertyFunc, postColumnFunc); if (statement == null) { throw new Exception(string.Format("Generate sql of \"{0}\" entity error", entity.GetType().FullName)); } var sqlValueBytes = ProtoBufUtils.Serialize(statement); string sqlQueueKey = SqlStatementManager.GetSqlQueueKey(statement.IdentityID); //修复顺序重复导致Sql执行乱序问题 sqlList.Add(new KeyValuePair <string, byte[]>(sqlQueueKey, sqlValueBytes)); TraceLog.WriteDebug($"Send to polls[{sqlQueueKey}]-> sql:{statement.ToString()}"); } } RedisConnectionPool.ProcessPipeline(p => { bool hasPost = false; var groupSqlList = sqlList.GroupBy(t => t.Key); int sqlCount = sqlList.Count; foreach (var g in groupSqlList) { var values = g.Select(t => t.Value).ToArray(); p.QueueCommand(client => ((RedisClient)client).LPush(g.Key, values), () => {//onSuccess ProfileManager.PostSqlOfMessageQueueTimes(g.Key, sqlCount); }); hasPost = true; } if (hasPost) { p.Flush(); } }); } catch (Exception ex) { TraceLog.WriteError("Send To Db key:{0} error:{1}", key, ex); } }