Пример #1
0
        /// <summary>
        /// Send entity to db saved.
        /// </summary>
        /// <param name="postColumnFunc"></param>
        /// <param name="getPropertyFunc"></param>
        /// <param name="entityList"></param>
        public static void SendToDb <T>(EntityPropertyGetFunc <T> getPropertyFunc, EnttiyPostColumnFunc <T> postColumnFunc, params T[] entityList) where T : ISqlEntity
        {
            string key = "";

            try
            {
                if (entityList == null || entityList.Length == 0)
                {
                    return;
                }
                var sender    = new SqlDataSender(false);
                var groupList = entityList.GroupBy(t => t.GetMessageQueueId());
                var sqlList   = new List <KeyValuePair <string, KeyValuePair <byte[], long> > >();

                foreach (var g in groupList)
                {
                    key = g.Key.ToString();
                    var valueList = g.ToList();

                    foreach (var entity in valueList)
                    {
                        if (entity == null)
                        {
                            continue;
                        }

                        SqlStatement statement = sender.GenerateSqlQueue <T>(entity, getPropertyFunc, postColumnFunc);
                        if (statement == null)
                        {
                            throw new Exception(string.Format("Generate sql of \"{0}\" entity error", entity.GetType().FullName));
                        }

                        var    sqlValueBytes = ProtoBufUtils.Serialize(statement);
                        string sqlQueueKey   = SqlStatementManager.GetSqlQueueKey(statement.IdentityID);
                        sqlList.Add(new KeyValuePair <string, KeyValuePair <byte[], long> >(sqlQueueKey,
                                                                                            new KeyValuePair <byte[], long>(sqlValueBytes, DateTime.Now.Ticks)));
                    }
                }
                RedisConnectionPool.ProcessPipeline(p =>
                {
                    bool hasPost     = false;
                    var groupSqlList = sqlList.GroupBy(t => t.Key);
                    foreach (var g in groupSqlList)
                    {
                        var pairs = g.Select(t => t.Value).ToList();
                        p.QueueCommand(client => ((RedisClient)client).ZAdd(g.Key, pairs));
                        hasPost = true;
                    }
                    if (hasPost)
                    {
                        p.Flush();
                    }
                });
            }
            catch (Exception ex)
            {
                TraceLog.WriteError("Send To Db key:{0} error:{1}", key, ex);
            }
        }
Пример #2
0
        /// <summary>
        /// Send entity to db saved.
        /// </summary>
        /// <param name="entityList"></param>
        public static void SendToDb(params AbstractEntity[] entityList)
        {
            string key = "";

            try
            {
                if (entityList == null || entityList.Length == 0)
                {
                    return;
                }

                var sender    = new SqlDataSender();
                var groupList = entityList.GroupBy(t => t.GetIdentityId());
                var sqlList   = new List <KeyValuePair <string, KeyValuePair <byte[], long> > >();

                foreach (var g in groupList)
                {
                    var valueList = g.ToList();

                    foreach (var entity in valueList)
                    {
                        if (entity == null)
                        {
                            continue;
                        }

                        SqlStatement statement = sender.GenerateSqlQueue(entity);
                        if (statement == null)
                        {
                            throw new Exception(string.Format("Generate sql of \"{0}\" entity error", entity.GetType().FullName));
                        }

                        var    sqlValueBytes = ProtoBufUtils.Serialize(statement);
                        string sqlQueueKey   = SqlStatementManager.GetSqlQueueKey(statement.IdentityID);
                        sqlList.Add(new KeyValuePair <string, KeyValuePair <byte[], long> >(sqlQueueKey,
                                                                                            new KeyValuePair <byte[], long>(sqlValueBytes, DateTime.Now.Ticks)));
                    }
                }
                RedisConnectionPool.Process(client =>
                {
                    var groupSqlList = sqlList.GroupBy(t => t.Key);
                    foreach (var g in groupSqlList)
                    {
                        var pairs = g.Select(t => t.Value).ToList();
                        client.ZAdd(g.Key, pairs);
                    }
                });
            }
            catch (Exception ex)
            {
                TraceLog.WriteError("Send To Db key:{0} error:{1}", key, ex);
            }
        }
Пример #3
0
        /// <summary>
        /// Generate Sql statements from the Keys-Values
        /// </summary>
        /// <param name="sender"></param>
        /// <param name="client"></param>
        /// <param name="keys"></param>
        /// <param name="values"></param>
        /// <returns></returns>
        private static IEnumerable <KeyValuePair <string, KeyValuePair <byte[], long> > > GenerateSqlFrom(SqlDataSender sender, RedisClient client, byte[][] keys, byte[][] values)
        {
            var typeKeyValuePairs = new List <KeyValuePair <string, byte[][]> >();

            for (int i = 0; i < keys.Length; i++)
            {
                byte[]   keyBytes       = keys[i];
                byte[]   headBytes      = values[i];
                string   entityTypeKey  = RedisConnectionPool.ToStringKey(keyBytes);
                string[] entityKeys     = entityTypeKey.Split(',')[0].Split('_');
                string   typeName       = entityKeys[0];
                byte[]   entityKeyBytes = RedisConnectionPool.ToByteKey(entityKeys[1]);
                typeKeyValuePairs.Add(new KeyValuePair <string, byte[][]>(typeName, new[] { entityKeyBytes, keyBytes, headBytes })
                                      );
            }
            var sqlList   = new List <KeyValuePair <string, KeyValuePair <byte[], long> > >();
            var typeGroup = typeKeyValuePairs.GroupBy(p => p.Key);

            foreach (var g in typeGroup)
            {
                var typeName = g.Key;
                try
                {
                    string entityParentKey = RedisConnectionPool.GetRedisEntityKeyName(typeName);
                    string asmName         = "";
                    var    enitityAsm      = EntitySchemaSet.EntityAssembly;
                    if (enitityAsm != null)
                    {
                        asmName = "," + enitityAsm.GetName().Name;
                    }

                    Type type = Type.GetType(string.Format("{0}{1}", RedisConnectionPool.DecodeTypeName(typeName), asmName), false, true);
                    if (type == null)
                    {
                        //调试模式下type为空处理
                        type = enitityAsm != null?enitityAsm.GetType(RedisConnectionPool.DecodeTypeName(typeName), false, true) : null;

                        if (type == null)
                        {
                            throw new ArgumentTypeException(string.Format("Get entity \"{0}\" type is null", entityParentKey));
                        }
                    }
                    var keyBuffers   = g.Select(p => p.Value[0]).ToArray();
                    var headBuffers  = g.Select(p => p.Value[2]).ToArray();
                    var valueBuffers = client.HMGet(entityParentKey, keyBuffers);

                    for (int i = 0; i < keyBuffers.Length; i++)
                    {
                        string keyCode   = RedisConnectionPool.ToStringKey(keyBuffers[i]);
                        var    buffer    = valueBuffers != null && valueBuffers.Length > i ? valueBuffers[i] : null;
                        var    headBytes = headBuffers[i];
                        int    identity;
                        int    state;
                        DecodeHeadBytes(headBytes, out identity, out state);

                        AbstractEntity entity = null;
                        if (state == 1 && buffer == null)
                        {
                            //entity remove ops
                            entity          = type.CreateInstance(new object[0]) as AbstractEntity;
                            entity.IsDelete = true;
                            entity.SetKeyValue(keyCode, typeName);
                        }
                        else if (buffer != null)
                        {
                            entity = _serializer.Deserialize(buffer, type) as AbstractEntity;
                        }
                        if (entity != null)
                        {
                            if (state == 1)
                            {
                                entity.IsDelete = true;
                            }
                            SqlStatement statement = sender.GenerateSqlQueue(entity);
                            if (statement == null)
                            {
                                throw new Exception(string.Format("Generate sql of \"{0}\" entity error", typeName));
                            }
                            var    sqlValueBytes = ProtoBufUtils.Serialize(statement);
                            string sqlQueueKey   = SqlStatementManager.GetSqlQueueKey(statement.IdentityID);
                            sqlList.Add(new KeyValuePair <string, KeyValuePair <byte[], long> >(sqlQueueKey,
                                                                                                new KeyValuePair <byte[], long>(sqlValueBytes, DateTime.Now.Ticks)));
                        }
                        else
                        {
                            throw new Exception(string.Format("Get \"{0}\" entity is null, keycode:{1},buffer len:{2},state:{3}",
                                                              typeName, keyCode, buffer == null ? -1 : buffer.Length, state));
                        }
                    }
                }
                catch (Exception er)
                {
                    TraceLog.WriteError("FindEntityFromRedis {0} error:{1}", typeName, er);
                    var errorKeys   = g.Select(p => p.Value[1]).ToArray();
                    var errorValues = g.Select(p => p.Value[2]).ToArray();
                    client.HMSet(SqlSyncWaitErrirQueueKey, errorKeys, errorValues);
                }
            }
            return(sqlList);
        }
Пример #4
0
        private static List<KeyValuePair<string, KeyValuePair<byte[], long>>> FindEntityFromRedis(SqlDataSender sender, RedisClient client, byte[][] keys, byte[][] values)
        {
            var typeKeyValuePairs = new List<KeyValuePair<string, byte[][]>>();
            for (int i = 0; i < keys.Length; i++)
            {
                byte[] keyBytes = keys[i];
                byte[] headBytes = values[i];
                string entityTypeKey = RedisConnectionPool.ToStringKey(keyBytes);
                string[] entityKeys = entityTypeKey.Split(',')[0].Split('_');
                string typeName = entityKeys[0];
                byte[] entityKeyBytes = RedisConnectionPool.ToByteKey(entityKeys[1]);
                typeKeyValuePairs.Add(
                    new KeyValuePair<string, byte[][]>(typeName, new[] { entityKeyBytes, keyBytes, headBytes })
                );
            }
            var sqlList = new List<KeyValuePair<string, KeyValuePair<byte[], long>>>();
            var typeGroup = typeKeyValuePairs.GroupBy(p => p.Key);
            foreach (var g in typeGroup)
            {
                var typeName = g.Key;
                try
                {
                    string entityParentKey = RedisConnectionPool.GetRedisEntityKeyName(typeName);
                    string asmName = "";
                    var enitityAsm = EntitySchemaSet.EntityAssembly;
                    if (enitityAsm != null)
                    {
                        asmName = "," + enitityAsm.GetName().Name;
                    }

                    Type type = Type.GetType(string.Format("{0}{1}", typeName, asmName), false, true);
                    if (type == null)
                    {
                        //调试模式下type为空处理
                        type = enitityAsm.GetType(typeName, false, true);
                        if (type == null)
                        {
                            throw new ArgumentTypeException(string.Format("Get entity \"{0}\" type is null", entityParentKey));
                        }
                    }
                    var keyBuffers = g.Select(p => p.Value[0]).ToArray();
                    var headBuffers = g.Select(p => p.Value[2]).ToArray();
                    var valueBuffers = client.HMGet(entityParentKey, keyBuffers);

                    for (int i = 0; i < keyBuffers.Length; i++)
                    {
                        string keyCode = RedisConnectionPool.ToStringKey(keyBuffers[i]);
                        var buffer = valueBuffers[i];
                        var headBytes = headBuffers[i];
                        int identity;
                        int state;
                        DecodeHeadBytes(headBytes, out identity, out state);

                        AbstractEntity entity = null;
                        if (state == 1 && buffer == null)
                        {
                            //entity remove ops
                            entity = type.CreateInstance(new object[0]) as AbstractEntity;
                            entity.IsDelete = true;
                            entity.SetKeyValue(keyCode, typeName);
                        }
                        else if (buffer != null)
                        {
                            entity = ProtoBufUtils.Deserialize(buffer, type) as AbstractEntity;
                        }
                        if (entity != null)
                        {
                            if (state == 1) entity.IsDelete = true;
                            SqlStatement statement = sender.GenerateSqlQueue(entity);
                            var sqlValueBytes = ProtoBufUtils.Serialize(statement);
                            string sqlQueueKey = SqlStatementManager.GetSqlQueueKey(statement.IdentityID);
                            sqlList.Add(new KeyValuePair<string, KeyValuePair<byte[], long>>(sqlQueueKey,
                                new KeyValuePair<byte[], long>(sqlValueBytes, DateTime.Now.Ticks)));
                        }
                        else
                        {
                            throw new Exception(string.Format("Get \"{0}\" entity is null, keycode:{1},buffer len:{2}",
                                typeName, keyCode, buffer == null ? -1 : buffer.Length));
                        }
                    }
                }
                catch (Exception er)
                {
                    TraceLog.WriteError("FindEntityFromRedis {0} error:{1}", typeName, er);
                    var errorKeys = g.Select(p => p.Value[1]).ToArray();
                    var errorValues = g.Select(p => p.Value[2]).ToArray();
                    client.HMSet(SqlSyncWaitErrirQueueKey, errorKeys, errorValues);

                }
            }
            return sqlList;
        }
Пример #5
0
        /// <summary>
        /// Send entity to db saved.
        /// </summary>
        /// <param name="postColumnFunc"></param>
        /// <param name="getPropertyFunc"></param>
        /// <param name="entityList"></param>
        public static void SendToDb <T>(string connectKey, EntityPropertyGetFunc <T> getPropertyFunc, EnttiyPostColumnFunc <T> postColumnFunc, params T[] entityList) where T : ISqlEntity
        {
            string key = "";

            try
            {
                if (entityList == null || entityList.Length == 0)
                {
                    return;
                }
                var sender    = new SqlDataSender(false, connectKey);
                var groupList = entityList.GroupBy(t => t.GetMessageQueueId());
                var sqlList   = new List <KeyValuePair <string, byte[]> >();

                foreach (var g in groupList)
                {
                    key = g.Key.ToString();
                    var valueList = g.ToList();

                    foreach (var entity in valueList)
                    {
                        if (entity == null)
                        {
                            continue;
                        }
                        SqlStatement statement = sender.GenerateSqlQueue <T>(entity, getPropertyFunc, postColumnFunc);
                        if (statement == null)
                        {
                            throw new Exception(string.Format("Generate sql of \"{0}\" entity error", entity.GetType().FullName));
                        }

                        var    sqlValueBytes = ProtoBufUtils.Serialize(statement);
                        string sqlQueueKey   = SqlStatementManager.GetSqlQueueKey(statement.IdentityID);
                        //修复顺序重复导致Sql执行乱序问题
                        sqlList.Add(new KeyValuePair <string, byte[]>(sqlQueueKey, sqlValueBytes));

                        TraceLog.WriteDebug($"Send to polls[{sqlQueueKey}]-> sql:{statement.ToString()}");
                    }
                }
                RedisConnectionPool.ProcessPipeline(p =>
                {
                    bool hasPost     = false;
                    var groupSqlList = sqlList.GroupBy(t => t.Key);
                    int sqlCount     = sqlList.Count;
                    foreach (var g in groupSqlList)
                    {
                        var values = g.Select(t => t.Value).ToArray();
                        p.QueueCommand(client => ((RedisClient)client).LPush(g.Key, values), () =>
                        {//onSuccess
                            ProfileManager.PostSqlOfMessageQueueTimes(g.Key, sqlCount);
                        });
                        hasPost = true;
                    }
                    if (hasPost)
                    {
                        p.Flush();
                    }
                });
            }
            catch (Exception ex)
            {
                TraceLog.WriteError("Send To Db key:{0} error:{1}", key, ex);
            }
        }
Пример #6
0
        /// <summary>
        /// Send entity to db saved.
        /// </summary>
        /// <param name="entityList"></param>
        public static void SendToDb(params AbstractEntity[] entityList)
        {
            string key = "";
            try
            {
                if (entityList == null || entityList.Length == 0) return;

                var sender = new SqlDataSender();
                var groupList = entityList.GroupBy(t => t.GetIdentityId());
                var sqlList = new List<KeyValuePair<string, KeyValuePair<byte[], long>>>();

                foreach (var g in groupList)
                {
                    var valueList = g.ToList();

                    foreach (var entity in valueList)
                    {
                        if (entity == null) continue;

                        SqlStatement statement = sender.GenerateSqlQueue(entity);
                        if (statement == null)
                        {
                            throw new Exception(string.Format("Generate sql of \"{0}\" entity error", entity.GetType().FullName));
                        }

                        var sqlValueBytes = ProtoBufUtils.Serialize(statement);
                        string sqlQueueKey = SqlStatementManager.GetSqlQueueKey(statement.IdentityID);
                        sqlList.Add(new KeyValuePair<string, KeyValuePair<byte[], long>>(sqlQueueKey,
                            new KeyValuePair<byte[], long>(sqlValueBytes, DateTime.Now.Ticks)));
                    }
                }
                RedisConnectionPool.Process(client =>
                {
                    var groupSqlList = sqlList.GroupBy(t => t.Key);
                    foreach (var g in groupSqlList)
                    {
                        var pairs = g.Select(t => t.Value).ToList();
                        client.ZAdd(g.Key, pairs);
                    }
                });
            }
            catch (Exception ex)
            {
                TraceLog.WriteError("Send To Db key:{0} error:{1}", key, ex);
            }
        }