/// <summary> /// Send entity to db saved. /// </summary> /// <param name="postColumnFunc"></param> /// <param name="getPropertyFunc"></param> /// <param name="entityList"></param> public static void SendToDb <T>(EntityPropertyGetFunc <T> getPropertyFunc, EnttiyPostColumnFunc <T> postColumnFunc, params T[] entityList) where T : ISqlEntity { string key = ""; try { if (entityList == null || entityList.Length == 0) { return; } var sender = new SqlDataSender(false); var groupList = entityList.GroupBy(t => t.GetMessageQueueId()); var sqlList = new List <KeyValuePair <string, KeyValuePair <byte[], long> > >(); foreach (var g in groupList) { key = g.Key.ToString(); var valueList = g.ToList(); foreach (var entity in valueList) { if (entity == null) { continue; } SqlStatement statement = sender.GenerateSqlQueue <T>(entity, getPropertyFunc, postColumnFunc); if (statement == null) { throw new Exception(string.Format("Generate sql of \"{0}\" entity error", entity.GetType().FullName)); } var sqlValueBytes = ProtoBufUtils.Serialize(statement); string sqlQueueKey = SqlStatementManager.GetSqlQueueKey(statement.IdentityID); sqlList.Add(new KeyValuePair <string, KeyValuePair <byte[], long> >(sqlQueueKey, new KeyValuePair <byte[], long>(sqlValueBytes, DateTime.Now.Ticks))); } } RedisConnectionPool.ProcessPipeline(p => { bool hasPost = false; var groupSqlList = sqlList.GroupBy(t => t.Key); foreach (var g in groupSqlList) { var pairs = g.Select(t => t.Value).ToList(); p.QueueCommand(client => ((RedisClient)client).ZAdd(g.Key, pairs)); hasPost = true; } if (hasPost) { p.Flush(); } }); } catch (Exception ex) { TraceLog.WriteError("Send To Db key:{0} error:{1}", key, ex); } }
/// <summary> /// Send entity to db saved. /// </summary> /// <param name="entityList"></param> public static void SendToDb(params AbstractEntity[] entityList) { string key = ""; try { if (entityList == null || entityList.Length == 0) { return; } var sender = new SqlDataSender(); var groupList = entityList.GroupBy(t => t.GetIdentityId()); var sqlList = new List <KeyValuePair <string, KeyValuePair <byte[], long> > >(); foreach (var g in groupList) { var valueList = g.ToList(); foreach (var entity in valueList) { if (entity == null) { continue; } SqlStatement statement = sender.GenerateSqlQueue(entity); if (statement == null) { throw new Exception(string.Format("Generate sql of \"{0}\" entity error", entity.GetType().FullName)); } var sqlValueBytes = ProtoBufUtils.Serialize(statement); string sqlQueueKey = SqlStatementManager.GetSqlQueueKey(statement.IdentityID); sqlList.Add(new KeyValuePair <string, KeyValuePair <byte[], long> >(sqlQueueKey, new KeyValuePair <byte[], long>(sqlValueBytes, DateTime.Now.Ticks))); } } RedisConnectionPool.Process(client => { var groupSqlList = sqlList.GroupBy(t => t.Key); foreach (var g in groupSqlList) { var pairs = g.Select(t => t.Value).ToList(); client.ZAdd(g.Key, pairs); } }); } catch (Exception ex) { TraceLog.WriteError("Send To Db key:{0} error:{1}", key, ex); } }
/// <summary> /// Generate Sql statements from the Keys-Values /// </summary> /// <param name="sender"></param> /// <param name="client"></param> /// <param name="keys"></param> /// <param name="values"></param> /// <returns></returns> private static IEnumerable <KeyValuePair <string, KeyValuePair <byte[], long> > > GenerateSqlFrom(SqlDataSender sender, RedisClient client, byte[][] keys, byte[][] values) { var typeKeyValuePairs = new List <KeyValuePair <string, byte[][]> >(); for (int i = 0; i < keys.Length; i++) { byte[] keyBytes = keys[i]; byte[] headBytes = values[i]; string entityTypeKey = RedisConnectionPool.ToStringKey(keyBytes); string[] entityKeys = entityTypeKey.Split(',')[0].Split('_'); string typeName = entityKeys[0]; byte[] entityKeyBytes = RedisConnectionPool.ToByteKey(entityKeys[1]); typeKeyValuePairs.Add(new KeyValuePair <string, byte[][]>(typeName, new[] { entityKeyBytes, keyBytes, headBytes }) ); } var sqlList = new List <KeyValuePair <string, KeyValuePair <byte[], long> > >(); var typeGroup = typeKeyValuePairs.GroupBy(p => p.Key); foreach (var g in typeGroup) { var typeName = g.Key; try { string entityParentKey = RedisConnectionPool.GetRedisEntityKeyName(typeName); string asmName = ""; var enitityAsm = EntitySchemaSet.EntityAssembly; if (enitityAsm != null) { asmName = "," + enitityAsm.GetName().Name; } Type type = Type.GetType(string.Format("{0}{1}", RedisConnectionPool.DecodeTypeName(typeName), asmName), false, true); if (type == null) { //调试模式下type为空处理 type = enitityAsm != null?enitityAsm.GetType(RedisConnectionPool.DecodeTypeName(typeName), false, true) : null; if (type == null) { throw new ArgumentTypeException(string.Format("Get entity \"{0}\" type is null", entityParentKey)); } } var keyBuffers = g.Select(p => p.Value[0]).ToArray(); var headBuffers = g.Select(p => p.Value[2]).ToArray(); var valueBuffers = client.HMGet(entityParentKey, keyBuffers); for (int i = 0; i < keyBuffers.Length; i++) { string keyCode = RedisConnectionPool.ToStringKey(keyBuffers[i]); var buffer = valueBuffers != null && valueBuffers.Length > i ? valueBuffers[i] : null; var headBytes = headBuffers[i]; int identity; int state; DecodeHeadBytes(headBytes, out identity, out state); AbstractEntity entity = null; if (state == 1 && buffer == null) { //entity remove ops entity = type.CreateInstance(new object[0]) as AbstractEntity; entity.IsDelete = true; entity.SetKeyValue(keyCode, typeName); } else if (buffer != null) { entity = _serializer.Deserialize(buffer, type) as AbstractEntity; } if (entity != null) { if (state == 1) { entity.IsDelete = true; } SqlStatement statement = sender.GenerateSqlQueue(entity); if (statement == null) { throw new Exception(string.Format("Generate sql of \"{0}\" entity error", typeName)); } var sqlValueBytes = ProtoBufUtils.Serialize(statement); string sqlQueueKey = SqlStatementManager.GetSqlQueueKey(statement.IdentityID); sqlList.Add(new KeyValuePair <string, KeyValuePair <byte[], long> >(sqlQueueKey, new KeyValuePair <byte[], long>(sqlValueBytes, DateTime.Now.Ticks))); } else { throw new Exception(string.Format("Get \"{0}\" entity is null, keycode:{1},buffer len:{2},state:{3}", typeName, keyCode, buffer == null ? -1 : buffer.Length, state)); } } } catch (Exception er) { TraceLog.WriteError("FindEntityFromRedis {0} error:{1}", typeName, er); var errorKeys = g.Select(p => p.Value[1]).ToArray(); var errorValues = g.Select(p => p.Value[2]).ToArray(); client.HMSet(SqlSyncWaitErrirQueueKey, errorKeys, errorValues); } } return(sqlList); }
private static List<KeyValuePair<string, KeyValuePair<byte[], long>>> FindEntityFromRedis(SqlDataSender sender, RedisClient client, byte[][] keys, byte[][] values) { var typeKeyValuePairs = new List<KeyValuePair<string, byte[][]>>(); for (int i = 0; i < keys.Length; i++) { byte[] keyBytes = keys[i]; byte[] headBytes = values[i]; string entityTypeKey = RedisConnectionPool.ToStringKey(keyBytes); string[] entityKeys = entityTypeKey.Split(',')[0].Split('_'); string typeName = entityKeys[0]; byte[] entityKeyBytes = RedisConnectionPool.ToByteKey(entityKeys[1]); typeKeyValuePairs.Add( new KeyValuePair<string, byte[][]>(typeName, new[] { entityKeyBytes, keyBytes, headBytes }) ); } var sqlList = new List<KeyValuePair<string, KeyValuePair<byte[], long>>>(); var typeGroup = typeKeyValuePairs.GroupBy(p => p.Key); foreach (var g in typeGroup) { var typeName = g.Key; try { string entityParentKey = RedisConnectionPool.GetRedisEntityKeyName(typeName); string asmName = ""; var enitityAsm = EntitySchemaSet.EntityAssembly; if (enitityAsm != null) { asmName = "," + enitityAsm.GetName().Name; } Type type = Type.GetType(string.Format("{0}{1}", typeName, asmName), false, true); if (type == null) { //调试模式下type为空处理 type = enitityAsm.GetType(typeName, false, true); if (type == null) { throw new ArgumentTypeException(string.Format("Get entity \"{0}\" type is null", entityParentKey)); } } var keyBuffers = g.Select(p => p.Value[0]).ToArray(); var headBuffers = g.Select(p => p.Value[2]).ToArray(); var valueBuffers = client.HMGet(entityParentKey, keyBuffers); for (int i = 0; i < keyBuffers.Length; i++) { string keyCode = RedisConnectionPool.ToStringKey(keyBuffers[i]); var buffer = valueBuffers[i]; var headBytes = headBuffers[i]; int identity; int state; DecodeHeadBytes(headBytes, out identity, out state); AbstractEntity entity = null; if (state == 1 && buffer == null) { //entity remove ops entity = type.CreateInstance(new object[0]) as AbstractEntity; entity.IsDelete = true; entity.SetKeyValue(keyCode, typeName); } else if (buffer != null) { entity = ProtoBufUtils.Deserialize(buffer, type) as AbstractEntity; } if (entity != null) { if (state == 1) entity.IsDelete = true; SqlStatement statement = sender.GenerateSqlQueue(entity); var sqlValueBytes = ProtoBufUtils.Serialize(statement); string sqlQueueKey = SqlStatementManager.GetSqlQueueKey(statement.IdentityID); sqlList.Add(new KeyValuePair<string, KeyValuePair<byte[], long>>(sqlQueueKey, new KeyValuePair<byte[], long>(sqlValueBytes, DateTime.Now.Ticks))); } else { throw new Exception(string.Format("Get \"{0}\" entity is null, keycode:{1},buffer len:{2}", typeName, keyCode, buffer == null ? -1 : buffer.Length)); } } } catch (Exception er) { TraceLog.WriteError("FindEntityFromRedis {0} error:{1}", typeName, er); var errorKeys = g.Select(p => p.Value[1]).ToArray(); var errorValues = g.Select(p => p.Value[2]).ToArray(); client.HMSet(SqlSyncWaitErrirQueueKey, errorKeys, errorValues); } } return sqlList; }
/// <summary> /// Send entity to db saved. /// </summary> /// <param name="postColumnFunc"></param> /// <param name="getPropertyFunc"></param> /// <param name="entityList"></param> public static void SendToDb <T>(string connectKey, EntityPropertyGetFunc <T> getPropertyFunc, EnttiyPostColumnFunc <T> postColumnFunc, params T[] entityList) where T : ISqlEntity { string key = ""; try { if (entityList == null || entityList.Length == 0) { return; } var sender = new SqlDataSender(false, connectKey); var groupList = entityList.GroupBy(t => t.GetMessageQueueId()); var sqlList = new List <KeyValuePair <string, byte[]> >(); foreach (var g in groupList) { key = g.Key.ToString(); var valueList = g.ToList(); foreach (var entity in valueList) { if (entity == null) { continue; } SqlStatement statement = sender.GenerateSqlQueue <T>(entity, getPropertyFunc, postColumnFunc); if (statement == null) { throw new Exception(string.Format("Generate sql of \"{0}\" entity error", entity.GetType().FullName)); } var sqlValueBytes = ProtoBufUtils.Serialize(statement); string sqlQueueKey = SqlStatementManager.GetSqlQueueKey(statement.IdentityID); //修复顺序重复导致Sql执行乱序问题 sqlList.Add(new KeyValuePair <string, byte[]>(sqlQueueKey, sqlValueBytes)); TraceLog.WriteDebug($"Send to polls[{sqlQueueKey}]-> sql:{statement.ToString()}"); } } RedisConnectionPool.ProcessPipeline(p => { bool hasPost = false; var groupSqlList = sqlList.GroupBy(t => t.Key); int sqlCount = sqlList.Count; foreach (var g in groupSqlList) { var values = g.Select(t => t.Value).ToArray(); p.QueueCommand(client => ((RedisClient)client).LPush(g.Key, values), () => {//onSuccess ProfileManager.PostSqlOfMessageQueueTimes(g.Key, sqlCount); }); hasPost = true; } if (hasPost) { p.Flush(); } }); } catch (Exception ex) { TraceLog.WriteError("Send To Db key:{0} error:{1}", key, ex); } }
/// <summary> /// Send entity to db saved. /// </summary> /// <param name="entityList"></param> public static void SendToDb(params AbstractEntity[] entityList) { string key = ""; try { if (entityList == null || entityList.Length == 0) return; var sender = new SqlDataSender(); var groupList = entityList.GroupBy(t => t.GetIdentityId()); var sqlList = new List<KeyValuePair<string, KeyValuePair<byte[], long>>>(); foreach (var g in groupList) { var valueList = g.ToList(); foreach (var entity in valueList) { if (entity == null) continue; SqlStatement statement = sender.GenerateSqlQueue(entity); if (statement == null) { throw new Exception(string.Format("Generate sql of \"{0}\" entity error", entity.GetType().FullName)); } var sqlValueBytes = ProtoBufUtils.Serialize(statement); string sqlQueueKey = SqlStatementManager.GetSqlQueueKey(statement.IdentityID); sqlList.Add(new KeyValuePair<string, KeyValuePair<byte[], long>>(sqlQueueKey, new KeyValuePair<byte[], long>(sqlValueBytes, DateTime.Now.Ticks))); } } RedisConnectionPool.Process(client => { var groupSqlList = sqlList.GroupBy(t => t.Key); foreach (var g in groupSqlList) { var pairs = g.Select(t => t.Value).ToList(); client.ZAdd(g.Key, pairs); } }); } catch (Exception ex) { TraceLog.WriteError("Send To Db key:{0} error:{1}", key, ex); } }