/// <summary> /// 从Redis内存移除,并保存到数据库, /// </summary> /// <param name="match">实体类型, 实体Key列表</param> public static void RemoveToDatabase(params KeyValuePair <Type, IList <string> >[] match) { var removeEntityKeys = new List <KeyValuePair <string, byte[][]> >(); var entityList = new List <EntityHistory>(); RedisConnectionPool.ProcessReadOnly(client => { foreach (var express in match) { try { string hashtId = RedisConnectionPool.GetRedisEntityKeyName(express.Key); byte[][] keyBytes = express.Value.Select(t => RedisConnectionPool.ToByteKey(t)).ToArray(); if (keyBytes.Length == 0) { continue; } removeEntityKeys.Add(new KeyValuePair <string, byte[][]>(hashtId, keyBytes)); //转存到DB使用protobuf byte[][] valueBytes = client.HMGet(hashtId, keyBytes); for (int i = 0; i < keyBytes.Length; i++) { entityList.Add(new EntityHistory() { Key = string.Format("{0}_{1}", hashtId, RedisConnectionPool.ToStringKey(keyBytes[i])), Value = valueBytes[i] }); } } catch (Exception ex) { TraceLog.WriteError("Redis cache remove key:{0} to Database error:{1}", express, ex); } } }); if (entityList.Count > 0) { DataSyncManager.SendSql <EntityHistory>(entityList, false, true); RedisConnectionPool.ProcessReadOnly(client => { foreach (var pair in removeEntityKeys) { client.HDel(pair.Key, pair.Value); } }); } }
/// <summary> /// Process synchronized queue of redis /// </summary> /// <param name="sysnWorkingQueueKey"></param> /// <param name="keys"></param> /// <param name="values"></param> private static void DoProcessRedisSyncQueue(string sysnWorkingQueueKey, byte[][] keys, byte[][] values) { try { var redisSyncErrorQueue = new List <byte[][]>(); var setList = new List <KeyValuePair <string, byte[][]> >(); var removeList = new List <KeyValuePair <string, byte[]> >(); var sqlWaitSyncQueue = new List <KeyValuePair <string, byte[][]> >(); for (int i = 0; i < keys.Length; i++) { byte[] keyBytes = keys[i]; byte[] valueBytes = values[i]; try { string[] queueKey = RedisConnectionPool.ToStringKey(keyBytes).Split('_'); string entityTypeName = RedisConnectionPool.DecodeTypeName(queueKey[0]); string entityParentKey = RedisConnectionPool.GetRedisEntityKeyName(queueKey[0]); byte[] entityKeyBytes = RedisConnectionPool.ToByteKey(queueKey[1]); byte[] headBytes; byte[] entityValBytes; int state; int identity; DecodeValueBytes(valueBytes, out headBytes, out entityValBytes, out state, out identity); if (state == 1) { removeList.Add(new KeyValuePair <string, byte[]>(entityParentKey, entityKeyBytes)); } else { setList.Add(new KeyValuePair <string, byte[][]>(entityParentKey, new[] { entityKeyBytes, entityValBytes })); } bool isStoreInDb = true; SchemaTable schema; if (EntitySchemaSet.TryGet(entityTypeName, out schema)) { isStoreInDb = schema.StorageType.HasFlag(StorageType.WriteOnlyDB) || schema.StorageType.HasFlag(StorageType.ReadWriteDB); } if (_enableWriteToDb && isStoreInDb) { //增加到Sql等待队列 string sqlWaitQueueKey = GetSqlWaitSyncQueueKey(identity); sqlWaitSyncQueue.Add(new KeyValuePair <string, byte[][]>(sqlWaitQueueKey, new[] { keyBytes, headBytes })); } } catch { redisSyncErrorQueue.Add(new[] { keyBytes, valueBytes }); } } var redisErrorKeys = redisSyncErrorQueue.Select(p => p[0]).ToArray(); var redisErrorValues = redisSyncErrorQueue.Select(p => p[1]).ToArray(); var sqlWaitGroups = sqlWaitSyncQueue.GroupBy(p => p.Key); var setGroups = setList.GroupBy(p => p.Key); var removeGroups = removeList.GroupBy(p => p.Key); RedisConnectionPool.Process(client => { foreach (var g in setGroups) { var entityKeys = g.Select(p => p.Value[0]).ToArray(); var entityValues = g.Select(p => p.Value[1]).ToArray(); client.HMSet(g.Key, entityKeys, entityValues); } foreach (var g in removeGroups) { var keybytes = g.Select(p => p.Value).ToArray(); client.HDel(g.Key, keybytes); } if (redisErrorKeys.Length > 0) { client.HMSet(RedisSyncErrorQueueKey, redisErrorKeys, redisErrorValues); } foreach (var g in sqlWaitGroups) { var sqlWaitKeys = g.Select(p => p.Value[0]).ToArray(); var sqlWaitValues = g.Select(p => p.Value[1]).ToArray(); client.HMSet(g.Key, sqlWaitKeys, sqlWaitValues); } }); } catch (Exception ex) { TraceLog.WriteError("DoProcessRedisSyncQueue error:{0}", ex); try { RedisConnectionPool.Process(client => client.HMSet(RedisSyncErrorQueueKey, keys, values)); } catch (Exception er) { TraceLog.WriteError("Put RedisSyncErrorQueue error:{0}", er); } } }
/// <summary> /// Generate Sql statements from the Keys-Values /// </summary> /// <param name="sender"></param> /// <param name="client"></param> /// <param name="keys"></param> /// <param name="values"></param> /// <returns></returns> private static IEnumerable <KeyValuePair <string, KeyValuePair <byte[], long> > > GenerateSqlFrom(SqlDataSender sender, RedisClient client, byte[][] keys, byte[][] values) { var typeKeyValuePairs = new List <KeyValuePair <string, byte[][]> >(); for (int i = 0; i < keys.Length; i++) { byte[] keyBytes = keys[i]; byte[] headBytes = values[i]; string entityTypeKey = RedisConnectionPool.ToStringKey(keyBytes); string[] entityKeys = entityTypeKey.Split(',')[0].Split('_'); string typeName = entityKeys[0]; byte[] entityKeyBytes = RedisConnectionPool.ToByteKey(entityKeys[1]); typeKeyValuePairs.Add(new KeyValuePair <string, byte[][]>(typeName, new[] { entityKeyBytes, keyBytes, headBytes }) ); } var sqlList = new List <KeyValuePair <string, KeyValuePair <byte[], long> > >(); var typeGroup = typeKeyValuePairs.GroupBy(p => p.Key); foreach (var g in typeGroup) { var typeName = g.Key; try { string entityParentKey = RedisConnectionPool.GetRedisEntityKeyName(typeName); string asmName = ""; var enitityAsm = EntitySchemaSet.EntityAssembly; if (enitityAsm != null) { asmName = "," + enitityAsm.GetName().Name; } Type type = Type.GetType(string.Format("{0}{1}", RedisConnectionPool.DecodeTypeName(typeName), asmName), false, true); if (type == null) { //调试模式下type为空处理 type = enitityAsm != null?enitityAsm.GetType(RedisConnectionPool.DecodeTypeName(typeName), false, true) : null; if (type == null) { throw new ArgumentTypeException(string.Format("Get entity \"{0}\" type is null", entityParentKey)); } } var keyBuffers = g.Select(p => p.Value[0]).ToArray(); var headBuffers = g.Select(p => p.Value[2]).ToArray(); var valueBuffers = client.HMGet(entityParentKey, keyBuffers); for (int i = 0; i < keyBuffers.Length; i++) { string keyCode = RedisConnectionPool.ToStringKey(keyBuffers[i]); var buffer = valueBuffers != null && valueBuffers.Length > i ? valueBuffers[i] : null; var headBytes = headBuffers[i]; int identity; int state; DecodeHeadBytes(headBytes, out identity, out state); AbstractEntity entity = null; if (state == 1 && buffer == null) { //entity remove ops entity = type.CreateInstance(new object[0]) as AbstractEntity; entity.IsDelete = true; entity.SetKeyValue(keyCode, typeName); } else if (buffer != null) { entity = _serializer.Deserialize(buffer, type) as AbstractEntity; } if (entity != null) { if (state == 1) { entity.IsDelete = true; } SqlStatement statement = sender.GenerateSqlQueue(entity); if (statement == null) { throw new Exception(string.Format("Generate sql of \"{0}\" entity error", typeName)); } var sqlValueBytes = ProtoBufUtils.Serialize(statement); string sqlQueueKey = SqlStatementManager.GetSqlQueueKey(statement.IdentityID); sqlList.Add(new KeyValuePair <string, KeyValuePair <byte[], long> >(sqlQueueKey, new KeyValuePair <byte[], long>(sqlValueBytes, DateTime.Now.Ticks))); } else { throw new Exception(string.Format("Get \"{0}\" entity is null, keycode:{1},buffer len:{2},state:{3}", typeName, keyCode, buffer == null ? -1 : buffer.Length, state)); } } } catch (Exception er) { TraceLog.WriteError("FindEntityFromRedis {0} error:{1}", typeName, er); var errorKeys = g.Select(p => p.Value[1]).ToArray(); var errorValues = g.Select(p => p.Value[2]).ToArray(); client.HMSet(SqlSyncWaitErrirQueueKey, errorKeys, errorValues); } } return(sqlList); }
/// <summary> /// Process synchronized queue of redis /// </summary> /// <param name="sysnWorkingQueueKey"></param> /// <param name="keys"></param> /// <param name="values"></param> private static bool ProcessRedisSyncQueue(string sysnWorkingQueueKey, byte[][] keys, byte[][] values) { bool result = false; try { var redisSyncErrorQueue = new List <byte[][]>(); var entityList = new List <RedisEntityItem>(); var entityRemList = new List <RedisEntityItem>(); var mutilKeyMapList = new List <RedisEntityItem>(); var mutilKeyMapRemList = new List <RedisEntityItem>(); var sqlWaitSyncQueue = new List <KeyValuePair <string, byte[][]> >(); for (int i = 0; i < keys.Length; i++) { byte[] keyBytes = keys[i]; byte[] valueBytes = values[i]; try { string[] queueKey = RedisConnectionPool.ToStringKey(keyBytes).Split('_'); string entityTypeName = RedisConnectionPool.DecodeTypeName(queueKey[0]); string entityParentKey = RedisConnectionPool.GetRedisEntityKeyName(queueKey[0]); byte[] entityKeyBytes = RedisConnectionPool.ToByteKey(queueKey[1]); bool hasMutilKey = false; bool isStoreInDb = true; SchemaTable schema; if (EntitySchemaSet.TryGet(entityTypeName, out schema)) { hasMutilKey = RedisConnectionPool.CurrRedisInfo.ClientVersion >= RedisStorageVersion.HashMutilKeyMap && schema.EntityType.IsSubclassOf(typeof(BaseEntity)) && schema.Keys.Length > 1; isStoreInDb = schema.StorageType.HasFlag(StorageType.WriteOnlyDB) || schema.StorageType.HasFlag(StorageType.ReadWriteDB); } byte[] headBytes; byte[] entityValBytes; int state; int identity; DecodeValueBytes(valueBytes, out headBytes, out entityValBytes, out state, out identity); var entityItem = new RedisEntityItem() { HashId = entityParentKey, UserId = identity, KeyBytes = entityKeyBytes, ValueBytes = entityValBytes, State = state, HasMutilKey = hasMutilKey }; if (entityItem.State == 1) { entityRemList.Add(entityItem); if (hasMutilKey) { mutilKeyMapRemList.Add(entityItem); } } else { entityList.Add(entityItem); if (hasMutilKey) { mutilKeyMapList.Add(entityItem); } } if (_enableWriteToDb && isStoreInDb) { //增加到Sql等待队列 string sqlWaitQueueKey = GetSqlWaitSyncQueueKey(identity); sqlWaitSyncQueue.Add(new KeyValuePair <string, byte[][]>(sqlWaitQueueKey, new[] { keyBytes, headBytes })); } } catch { redisSyncErrorQueue.Add(new[] { keyBytes, valueBytes }); } } var redisErrorKeys = redisSyncErrorQueue.Select(p => p[0]).ToArray(); var redisErrorValues = redisSyncErrorQueue.Select(p => p[1]).ToArray(); var sqlWaitGroups = sqlWaitSyncQueue.GroupBy(p => p.Key); var setGroups = entityList.GroupBy(p => p.HashId); var removeGroups = entityRemList.GroupBy(p => p.HashId); var mutilKeyMapGroups = mutilKeyMapList.GroupBy(p => p.HashId); var mutilKeyMapRemGroups = mutilKeyMapRemList.GroupBy(p => p.HashId); RedisConnectionPool.ProcessPipeline(pipeline => { bool hasPost = false; foreach (var g in setGroups) { var entityKeys = g.Select(p => p.KeyBytes).ToArray(); var entityValues = g.Select(p => p.ValueBytes).ToArray(); pipeline.QueueCommand(client => ((RedisClient)client).HMSet(g.Key, entityKeys, entityValues)); hasPost = true; } foreach (var g in removeGroups) { var keybytes = g.Select(p => p.KeyBytes).ToArray(); pipeline.QueueCommand(client => ((RedisClient)client).HDel(g.Key, keybytes)); hasPost = true; } foreach (var g in mutilKeyMapGroups) { string hashId = g.Key; var subGroup = g.GroupBy(t => t.UserId); foreach (var @group in subGroup) { string firstKey = AbstractEntity.EncodeKeyCode(@group.Key.ToString()); var keybytes = @group.Select(p => p.KeyBytes).ToArray(); pipeline.QueueCommand(client => RedisConnectionPool.SetMutilKeyMap((RedisClient)client, hashId, firstKey, keybytes)); hasPost = true; } } foreach (var g in mutilKeyMapRemGroups) { string hashId = g.Key; var subGroup = g.GroupBy(t => t.UserId); foreach (var @group in subGroup) { string firstKey = AbstractEntity.EncodeKeyCode(@group.Key.ToString()); var keybytes = @group.Select(p => p.KeyBytes).ToArray(); pipeline.QueueCommand(client => RedisConnectionPool.RemoveMutilKeyMap((RedisClient)client, hashId, firstKey, keybytes)); hasPost = true; } } if (redisErrorKeys.Length > 0) { pipeline.QueueCommand(client => ((RedisClient)client).HMSet(RedisSyncErrorQueueKey, redisErrorKeys, redisErrorValues)); hasPost = true; } foreach (var g in sqlWaitGroups) { var sqlWaitKeys = g.Select(p => p.Value[0]).ToArray(); var sqlWaitValues = g.Select(p => p.Value[1]).ToArray(); pipeline.QueueCommand(client => ((RedisClient)client).HMSet(g.Key, sqlWaitKeys, sqlWaitValues)); hasPost = true; } if (hasPost) { pipeline.Flush(); } result = redisErrorKeys.Length == 0; }); } catch (Exception ex) { TraceLog.WriteError("DoProcessRedisSyncQueue error:{0}", ex); try { RedisConnectionPool.Process(client => client.HMSet(RedisSyncErrorQueueKey, keys, values)); } catch (Exception er) { TraceLog.WriteError("Put RedisSyncErrorQueue error:{0}", er); } } return(result); }