/// <summary>
/// Send entity to db saved.
/// </summary>
/// <param name="postColumnFunc"></param>
/// <param name="getPropertyFunc"></param>
/// <param name="entityList"></param>
public static void SendToDb <T>(EntityPropertyGetFunc <T> getPropertyFunc, EnttiyPostColumnFunc <T> postColumnFunc, params T[] entityList) where T : ISqlEntity
{
string key = "";
try
{
if (entityList == null || entityList.Length == 0)
{
return;
}
var sender = new SqlDataSender(false);
var groupList = entityList.GroupBy(t => t.GetMessageQueueId());
var sqlList = new List <KeyValuePair <string, KeyValuePair <byte[], long> > >();
foreach (var g in groupList)
{
key = g.Key.ToString();
var valueList = g.ToList();
foreach (var entity in valueList)
{
if (entity == null)
{
continue;
}
SqlStatement statement = sender.GenerateSqlQueue <T>(entity, getPropertyFunc, postColumnFunc);
if (statement == null)
{
throw new Exception(string.Format("Generate sql of \"{0}\" entity error", entity.GetType().FullName));
}
var sqlValueBytes = ProtoBufUtils.Serialize(statement);
string sqlQueueKey = SqlStatementManager.GetSqlQueueKey(statement.IdentityID);
sqlList.Add(new KeyValuePair <string, KeyValuePair <byte[], long> >(sqlQueueKey,
new KeyValuePair <byte[], long>(sqlValueBytes, DateTime.Now.Ticks)));
}
}
RedisConnectionPool.ProcessPipeline(p =>
{
bool hasPost = false;
var groupSqlList = sqlList.GroupBy(t => t.Key);
foreach (var g in groupSqlList)
{
var pairs = g.Select(t => t.Value).ToList();
p.QueueCommand(client => ((RedisClient)client).ZAdd(g.Key, pairs));
hasPost = true;
}
if (hasPost)
{
p.Flush();
}
});
}
catch (Exception ex)
{
TraceLog.WriteError("Send To Db key:{0} error:{1}", key, ex);
}
}
/// <summary>
/// Process synchronized wait queue of Sql
/// </summary>
/// <param name="waitQueueKey"></param>
/// <param name="keys"></param>
/// <param name="values"></param>
private static void DoProcessSqlWaitSyncQueue(string waitQueueKey, byte[][] keys, byte[][] values)
{
try
{
var sqlSender = new SqlDataSender(false);
IEnumerable <KeyValuePair <string, KeyValuePair <byte[], long> > > sqlList = null;
RedisConnectionPool.ProcessPipeline(client =>
{
sqlList = GenerateSqlFrom(sqlSender, client, keys, values);
}, p =>
{
if (sqlList == null)
{
return(0);
}
var keyValuePairs = sqlList.ToList();
var groupSqlList = keyValuePairs.GroupBy(t => t.Key);
int sqlCount = keyValuePairs.Count();
bool hasPost = false;
long result = 0;
foreach (var g in groupSqlList)
{
var pairs = g.Select(t => t.Value).ToList();
p.QueueCommand(c => ((RedisClient)c).ZAdd(g.Key, pairs), r =>
{//onsuccess, 已经存在的返回值为0
result += r;
ProfileManager.PostSqlOfMessageQueueTimes(g.Key, sqlCount);
});
hasPost = true;
}
if (hasPost)
{
p.Flush();
}
return(result);
});
}
catch (Exception ex)
{
TraceLog.WriteError("DoProcessSqlWaitSyncQueue error:{0}", ex);
}
}
public void RedisPirpe()
{
RedisConnectionPool.ProcessPipeline(p =>
{
p.QueueCommand(c => c.Set("_test", "1"));
p.Flush();
});
RedisConnectionPool.ProcessPipeline(client =>
{
Assert.AreEqual(client.Get <string>("_test"), "1");
}, p =>
{
long result = 0;
p.QueueCommand(c =>
{
result = c.Set("_test", "2") ? 1 : 0;
});
p.Flush();
return(result);
});
}
/// <summary>
/// Process synchronized queue of redis
/// </summary>
/// <param name="sysnWorkingQueueKey"></param>
/// <param name="keys"></param>
/// <param name="values"></param>
private static bool ProcessRedisSyncQueue(string sysnWorkingQueueKey, byte[][] keys, byte[][] values)
{
bool result = false;
try
{
var redisSyncErrorQueue = new List <byte[][]>();
var entityList = new List <RedisEntityItem>();
var entityRemList = new List <RedisEntityItem>();
var mutilKeyMapList = new List <RedisEntityItem>();
var mutilKeyMapRemList = new List <RedisEntityItem>();
var sqlWaitSyncQueue = new List <KeyValuePair <string, byte[][]> >();
for (int i = 0; i < keys.Length; i++)
{
byte[] keyBytes = keys[i];
byte[] valueBytes = values[i];
try
{
string[] queueKey = RedisConnectionPool.ToStringKey(keyBytes).Split('_');
string entityTypeName = RedisConnectionPool.DecodeTypeName(queueKey[0]);
string entityParentKey = RedisConnectionPool.GetRedisEntityKeyName(queueKey[0]);
byte[] entityKeyBytes = RedisConnectionPool.ToByteKey(queueKey[1]);
bool hasMutilKey = false;
bool isStoreInDb = true;
SchemaTable schema;
if (EntitySchemaSet.TryGet(entityTypeName, out schema))
{
hasMutilKey = RedisConnectionPool.CurrRedisInfo.ClientVersion >= RedisStorageVersion.HashMutilKeyMap &&
schema.EntityType.IsSubclassOf(typeof(BaseEntity)) &&
schema.Keys.Length > 1;
isStoreInDb = schema.StorageType.HasFlag(StorageType.WriteOnlyDB) || schema.StorageType.HasFlag(StorageType.ReadWriteDB);
}
byte[] headBytes;
byte[] entityValBytes;
int state;
int identity;
DecodeValueBytes(valueBytes, out headBytes, out entityValBytes, out state, out identity);
var entityItem = new RedisEntityItem()
{
HashId = entityParentKey,
UserId = identity,
KeyBytes = entityKeyBytes,
ValueBytes = entityValBytes,
State = state,
HasMutilKey = hasMutilKey
};
if (entityItem.State == 1)
{
entityRemList.Add(entityItem);
if (hasMutilKey)
{
mutilKeyMapRemList.Add(entityItem);
}
}
else
{
entityList.Add(entityItem);
if (hasMutilKey)
{
mutilKeyMapList.Add(entityItem);
}
}
if (_enableWriteToDb && isStoreInDb)
{
//增加到Sql等待队列
string sqlWaitQueueKey = GetSqlWaitSyncQueueKey(identity);
sqlWaitSyncQueue.Add(new KeyValuePair <string, byte[][]>(sqlWaitQueueKey, new[] { keyBytes, headBytes }));
}
}
catch
{
redisSyncErrorQueue.Add(new[] { keyBytes, valueBytes });
}
}
var redisErrorKeys = redisSyncErrorQueue.Select(p => p[0]).ToArray();
var redisErrorValues = redisSyncErrorQueue.Select(p => p[1]).ToArray();
var sqlWaitGroups = sqlWaitSyncQueue.GroupBy(p => p.Key);
var setGroups = entityList.GroupBy(p => p.HashId);
var removeGroups = entityRemList.GroupBy(p => p.HashId);
var mutilKeyMapGroups = mutilKeyMapList.GroupBy(p => p.HashId);
var mutilKeyMapRemGroups = mutilKeyMapRemList.GroupBy(p => p.HashId);
RedisConnectionPool.ProcessPipeline(pipeline =>
{
bool hasPost = false;
foreach (var g in setGroups)
{
var entityKeys = g.Select(p => p.KeyBytes).ToArray();
var entityValues = g.Select(p => p.ValueBytes).ToArray();
pipeline.QueueCommand(client => ((RedisClient)client).HMSet(g.Key, entityKeys, entityValues));
hasPost = true;
}
foreach (var g in removeGroups)
{
var keybytes = g.Select(p => p.KeyBytes).ToArray();
pipeline.QueueCommand(client => ((RedisClient)client).HDel(g.Key, keybytes));
hasPost = true;
}
foreach (var g in mutilKeyMapGroups)
{
string hashId = g.Key;
var subGroup = g.GroupBy(t => t.UserId);
foreach (var @group in subGroup)
{
string firstKey = AbstractEntity.EncodeKeyCode(@group.Key.ToString());
var keybytes = @group.Select(p => p.KeyBytes).ToArray();
pipeline.QueueCommand(client => RedisConnectionPool.SetMutilKeyMap((RedisClient)client, hashId, firstKey, keybytes));
hasPost = true;
}
}
foreach (var g in mutilKeyMapRemGroups)
{
string hashId = g.Key;
var subGroup = g.GroupBy(t => t.UserId);
foreach (var @group in subGroup)
{
string firstKey = AbstractEntity.EncodeKeyCode(@group.Key.ToString());
var keybytes = @group.Select(p => p.KeyBytes).ToArray();
pipeline.QueueCommand(client => RedisConnectionPool.RemoveMutilKeyMap((RedisClient)client, hashId, firstKey, keybytes));
hasPost = true;
}
}
if (redisErrorKeys.Length > 0)
{
pipeline.QueueCommand(client => ((RedisClient)client).HMSet(RedisSyncErrorQueueKey, redisErrorKeys, redisErrorValues));
hasPost = true;
}
foreach (var g in sqlWaitGroups)
{
var sqlWaitKeys = g.Select(p => p.Value[0]).ToArray();
var sqlWaitValues = g.Select(p => p.Value[1]).ToArray();
pipeline.QueueCommand(client => ((RedisClient)client).HMSet(g.Key, sqlWaitKeys, sqlWaitValues));
hasPost = true;
}
if (hasPost)
{
pipeline.Flush();
}
result = redisErrorKeys.Length == 0;
});
}
catch (Exception ex)
{
TraceLog.WriteError("DoProcessRedisSyncQueue error:{0}", ex);
try
{
RedisConnectionPool.Process(client => client.HMSet(RedisSyncErrorQueueKey, keys, values));
}
catch (Exception er)
{
TraceLog.WriteError("Put RedisSyncErrorQueue error:{0}", er);
}
}
return(result);
}
private static void OnEntitySyncQueue(object state)
{
if (Interlocked.CompareExchange(ref _entityQueueRunning, 1, 0) == 0)
{
try
{
var tempRemove = Interlocked.Exchange(ref _entityRemoteSet, new HashSet <string>());
var temp = Interlocked.Exchange(ref _entitySet, new HashSet <string>());
if (temp.Count == 0 || _queueWatchTimers == null)
{
return;
}
TraceLog.WriteWarn("OnEntitySyncQueue execute count:{0}, success:{1}/total {2}, fail:{3} start...", temp.Count, ExecuteSuccessCount, SendWaitCount, ExecuteFailCount);
RedisConnectionPool.ProcessPipeline(pipeline =>
{
bool hasPost = false;
foreach (var key in temp)
{
var keyValues = key.Split('_', '|');
if (keyValues.Length != 3)
{
TraceLog.WriteWarn("OnEntitySyncQueue:{0}", key);
ExecuteFailCount++;
continue;
}
AbstractEntity entity = CacheFactory.GetPersonalEntity(key) as AbstractEntity;
int id = AbstractEntity.DecodeKeyCode(keyValues[1]).ToInt();
string keyCode = keyValues[2];
string redisKey = string.Format("{0}_{1}", keyValues[0], keyCode);
string hashId = GetRedisSyncQueueKey(id);
byte[] idBytes = BufferUtils.GetBytes(id);
var keyBytes = RedisConnectionPool.ToByteKey(redisKey);
bool isDelete;
byte[] entityBytes;
if (entity != null)
{
isDelete = entity.IsDelete;
entityBytes = _serializer.Serialize(entity);
//modify resean: set unchange status.
entity.Reset();
}
else if (tempRemove.Contains(key))
{
entityBytes = new byte[0];
isDelete = true;
}
else
{
ExecuteFailCount++;
TraceLog.WriteError("EntitySync queue key {0} faild object is null.", key);
continue;
}
byte[] stateBytes = BufferUtils.GetBytes(isDelete ? 1 : 0);
byte[] values =
BufferUtils.MergeBytes(BufferUtils.GetBytes(idBytes.Length + stateBytes.Length),
idBytes, stateBytes, entityBytes);
pipeline.QueueCommand(c =>
{
((RedisClient)c).HSet(hashId, keyBytes, values);
});
hasPost = true;
ExecuteSuccessCount++;
}
if (hasPost)
{
pipeline.Flush();
}
});
}
catch (Exception ex)
{
TraceLog.WriteError("EntitySync queue {0}", ex);
}
finally
{
Interlocked.Exchange(ref _entityQueueRunning, 0);
}
}
}
/// <summary>
/// Send entity to db saved.
/// </summary>
/// <param name="postColumnFunc"></param>
/// <param name="getPropertyFunc"></param>
/// <param name="entityList"></param>
public static void SendToDb <T>(string connectKey, EntityPropertyGetFunc <T> getPropertyFunc, EnttiyPostColumnFunc <T> postColumnFunc, params T[] entityList) where T : ISqlEntity
{
string key = "";
try
{
if (entityList == null || entityList.Length == 0)
{
return;
}
var sender = new SqlDataSender(false, connectKey);
var groupList = entityList.GroupBy(t => t.GetMessageQueueId());
var sqlList = new List <KeyValuePair <string, byte[]> >();
foreach (var g in groupList)
{
key = g.Key.ToString();
var valueList = g.ToList();
foreach (var entity in valueList)
{
if (entity == null)
{
continue;
}
SqlStatement statement = sender.GenerateSqlQueue <T>(entity, getPropertyFunc, postColumnFunc);
if (statement == null)
{
throw new Exception(string.Format("Generate sql of \"{0}\" entity error", entity.GetType().FullName));
}
var sqlValueBytes = ProtoBufUtils.Serialize(statement);
string sqlQueueKey = SqlStatementManager.GetSqlQueueKey(statement.IdentityID);
//修复顺序重复导致Sql执行乱序问题
sqlList.Add(new KeyValuePair <string, byte[]>(sqlQueueKey, sqlValueBytes));
TraceLog.WriteDebug($"Send to polls[{sqlQueueKey}]-> sql:{statement.ToString()}");
}
}
RedisConnectionPool.ProcessPipeline(p =>
{
bool hasPost = false;
var groupSqlList = sqlList.GroupBy(t => t.Key);
int sqlCount = sqlList.Count;
foreach (var g in groupSqlList)
{
var values = g.Select(t => t.Value).ToArray();
p.QueueCommand(client => ((RedisClient)client).LPush(g.Key, values), () =>
{//onSuccess
ProfileManager.PostSqlOfMessageQueueTimes(g.Key, sqlCount);
});
hasPost = true;
}
if (hasPost)
{
p.Flush();
}
});
}
catch (Exception ex)
{
TraceLog.WriteError("Send To Db key:{0} error:{1}", key, ex);
}
}
