/// <summary>
/// 放到消息队列池中
/// </summary>
/// <param name="statement"></param>
public static bool Put(SqlStatement statement)
{
bool result = false;
try
{
if (!IsUseSyncQueue)
{
return(false);
}
string tableName = statement.Table;
string key = GetSqlQueueKey(statement.IdentityID);
byte[] value = ProtoBufUtils.Serialize(statement);
RedisConnectionPool.Process(client =>
{
client.ZAdd(key, DateTime.Now.Ticks, value);
ProfileManager.PostSqlOfMessageQueueTimes(tableName, 1);
});
result = true;
}
catch (Exception ex)
{
TraceLog.WriteError("Sql update queue write error:{0}\r\n{1}", ex, JsonUtils.SerializeCustom(statement));
}
return(result);
}
public void RedisListTest()
{
var watch = Stopwatch.StartNew();
var keys = new List <string> {
"__List:1000:1", "__List:1000:2"
};
var script = @"
local result={}
local index = 0
local len = table.getn(KEYS)
for i=1, len do
local key = KEYS[i]
local values = redis.call('lrange',key,0,-1)
local l = table.getn(values)
result[i] = table.concat(values,',')
end
return result
";
List <string> valueList = null;
RedisConnectionPool.Process(c =>
{
valueList = c.ExecLuaAsList(script, keys.ToArray(), new string[0]);
});
Trace.WriteLine("value count:" + valueList.Count + "\r\n" + string.Join("\r\n", valueList));
WaitEnd(watch);
}
public void FindRedis()
{
RedisConnectionPool.Process(cli =>
{
/*
* var watch = Stopwatch.StartNew();
* key = "$FrameworkUnitTest.Cache.Model.UserKeyData";//"$FrameworkUnitTest.Cache.Model.QuestProcess";//
* //var keys = cli.HKeys(key);
* //Trace.WriteLine("load time:" + watch.ElapsedMilliseconds + "ms." + "keys:" + keys.Length);
* //var valueBytes = cli.HGet(key, keys);
* //Trace.WriteLine("load time:" + watch.ElapsedMilliseconds + "ms." + "vals:" + valueBytes.Length);
* byte[] suBytes = Encoding.Default.GetBytes("1380000");
* var script = @"
* local userId = KEYS[2]
* local key = KEYS[1]
* local keyBytes = redis.call('HKeys',key)
* print('keys:'.. table.getn(keyBytes))
* local valueBytes = {}
* for i,k in pairs(keyBytes) do
* if string.find(k, userId) then
* local val = redis.call('HGet',key, k)
* table.insert(valueBytes, val)
* end
* end
* return valueBytes
* ";
* var keybytes = Encoding.Default.GetBytes(key);
* var valueBytes = cli.Eval(script, 2, keybytes, suBytes);
* Trace.WriteLine("load time:" + watch.ElapsedMilliseconds + "ms." + "vals:" + (valueBytes.Length > 0 && valueBytes[0] != null ? Encoding.Default.GetString(valueBytes[0]) + "count:" + valueBytes.Length : "count:" + valueBytes.Length));
*
*/
});
}
/// <summary>
///
/// </summary>
/// <param name="token"></param>
/// <returns></returns>
public static UserToken GetUserToken(string token)
{
UserToken userToken = null;
string redisKey = string.Format("{0}@{1}", AccountServerToken, token);
if (!string.IsNullOrEmpty(RedisHost) && RedisConnected)
{
//use redis cache.
RedisConnectionPool.Process(client =>
{
var val = client.Get(redisKey);
userToken = val == null || val.Length == 0 ? new UserToken() : MathUtils.ParseJson <UserToken>(Encoding.UTF8.GetString(val));
});
return(userToken);
}
if (HttpContext.Current != null && HttpContext.Current.Cache != null)
{
userToken = (UserToken)HttpContext.Current.Cache[redisKey];
}
if (Equals(userToken, null) && userTokenCache.ContainsKey(redisKey))
{
userToken = (UserToken)userTokenCache[redisKey];
}
return(userToken);
}
public void CheckCompletedTest()
{
var keys = new[] { "testkey1", "testkey2" };
var vals = new[] { "1", "2" };
RedisConnectionPool.SetExpire(keys, vals, 300);
RedisConnectionPool.Process(c =>
{
var values = new List <string>();
using (var p = c.CreatePipeline())
{
foreach (var key in keys)
{
string k = key;
p.QueueCommand(cli => cli.Get <string>(k), v =>
{
values.Add(v);
});
}
p.Flush();
}
Trace.WriteLine(string.Join(",", values));
});
var watch = Stopwatch.StartNew();
var data = ShareCacheStruct.Get <ChildData>(1);
data.Age++;
bool result = CacheFactory.CheckCompleted();
Trace.WriteLine("Queue result:" + result);
WaitEnd(watch);
}
/// <summary>
///
/// </summary>
/// <param name="key"></param>
/// <param name="userToken"></param>
public static void SaveToken(string key, UserToken userToken)
{
string userKey = string.Format("{0}:{1}", AccountServerToken, userToken.UserId);
string redisKey = string.Format("{0}@{1}", AccountServerToken, key);
if (!string.IsNullOrEmpty(RedisHost) && RedisConnected)
{
try
{
string script = @"
local redisKey = KEYS[1]
local userKey = KEYS[2]
local key = KEYS[3]
local timeout = ARGV[1]
local val = ARGV[2]
local oldToken = redis.call('Get', userKey)
if oldToken and oldToken ~= nil then
local k = '__AccountToken@'..oldToken
redis.call('del', k)
end
redis.call('Set', redisKey, val, 'EX', timeout)
redis.call('Set', userKey, key, 'EX', timeout)
return 0
";
//use redis cache.
RedisConnectionPool.Process(client =>
{
int timeout = (int)(userToken.ExpireTime - DateTime.Now).TotalSeconds;
client.ExecLuaAsInt(script, new[] { redisKey, userKey, key }, new[] { timeout.ToString(), MathUtils.ToJson(userToken) });
});
}
catch (Exception ex)
{
TraceLog.WriteError("Save token errror:{0}", ex);
}
return;
}
if (HttpContext.Current != null && HttpContext.Current.Cache != null)
{
var cache = HttpContext.Current.Cache;
var oldToken = cache[userKey] as string;
if (!string.IsNullOrEmpty(oldToken))
{
cache.Remove(string.Format("{0}@{1}", AccountServerToken, oldToken));
}
cache[redisKey] = userToken;
cache[userKey] = key;
}
else
{
string oldToken;
if (userHashCache.TryGetValue(userToken.UserId, out oldToken))
{
UserToken temp;
userTokenCache.TryRemove(oldToken, out temp);
}
userTokenCache[key] = userToken;
}
}
private static void OnAsyncSend(string queueKey, byte[][] keyBytes, byte[][] valueBytes)
{
try
{
RedisConnectionPool.Process(client => client.HMSet(queueKey, keyBytes, valueBytes));
}
catch (Exception ex)
{
TraceLog.WriteError("Async send to redis's key:{0} error:{1}", queueKey, ex);
}
}
/// <summary>
/// Send entity to db saved.
/// </summary>
/// <param name="entityList"></param>
public static void SendToDb(params AbstractEntity[] entityList)
{
string key = "";
try
{
if (entityList == null || entityList.Length == 0)
{
return;
}
var sender = new SqlDataSender();
var groupList = entityList.GroupBy(t => t.GetIdentityId());
var sqlList = new List <KeyValuePair <string, KeyValuePair <byte[], long> > >();
foreach (var g in groupList)
{
var valueList = g.ToList();
foreach (var entity in valueList)
{
if (entity == null)
{
continue;
}
SqlStatement statement = sender.GenerateSqlQueue(entity);
if (statement == null)
{
throw new Exception(string.Format("Generate sql of \"{0}\" entity error", entity.GetType().FullName));
}
var sqlValueBytes = ProtoBufUtils.Serialize(statement);
string sqlQueueKey = SqlStatementManager.GetSqlQueueKey(statement.IdentityID);
sqlList.Add(new KeyValuePair <string, KeyValuePair <byte[], long> >(sqlQueueKey,
new KeyValuePair <byte[], long>(sqlValueBytes, DateTime.Now.Ticks)));
}
}
RedisConnectionPool.Process(client =>
{
var groupSqlList = sqlList.GroupBy(t => t.Key);
foreach (var g in groupSqlList)
{
var pairs = g.Select(t => t.Value).ToList();
client.ZAdd(g.Key, pairs);
}
});
}
catch (Exception ex)
{
TraceLog.WriteError("Send To Db key:{0} error:{1}", key, ex);
}
}
private static void SaveTo()
{
try
{
byte[] data = ProtoBufUtils.Serialize(_globalSession);
RedisConnectionPool.Process(client => client.Set(sessionRedisKey, data));
}
catch (Exception er)
{
TraceLog.WriteError("Save session to redis faild,{0}", er);
}
}
/// <summary>
/// 通过Redis键获取实体对象
/// </summary>
/// <param name="key"></param>
/// <param name="isRemove"></param>
/// <param name="type"></param>
/// <returns></returns>
public static dynamic GetEntityFromRedis(string key, bool isRemove, Type type = null)
{
string typeName;
string asmName;
bool isEntityType;
string redisKey;
string entityKey = GetEntityTypeFromKey(key, out typeName, ref type, out asmName, out isEntityType, out redisKey);
dynamic entity = null;
RedisConnectionPool.Process(client =>
{
if (isEntityType)
{
var data = client.Get <byte[]>(redisKey);
if (data != null && type != null)
{
entity = ProtoBufUtils.Deserialize(data, type);
}
}
else
{
var data = client.Get <byte[]>(redisKey);
if (data != null && type != null)
{
var dict = (IDictionary)ProtoBufUtils.Deserialize(data, type);
entity = dict[entityKey];
}
}
if (isRemove && entity == null && type != null)
{
string setId = (isEntityType ? typeName : redisKey) + ":remove";
var data = client.Get <byte[]>(setId);
if (data != null)
{
var dict = (IDictionary)ProtoBufUtils.Deserialize(data, type);
entity = dict[entityKey];
dict.Remove(entityKey);
if (dict.Count > 0)
{
client.Set(setId, ProtoBufUtils.Serialize(dict));
}
else
{
client.Remove(setId);
}
}
}
});
return(entity);
}
/// <summary>
///
/// </summary>
/// <param name="token"></param>
/// <param name="redisHost"></param>
/// <param name="redisDb"></param>
/// <returns></returns>
public static UserToken GetUserToken(string token, string redisHost, int redisDb)
{
UserToken userToken = null;
string redisKey = string.Format("{0}@{1}", AccountServerToken, token);
RedisConnectionPool.Process(client =>
{
var val = client.Get(redisKey);
userToken = val == null || val.Length == 0 ? new UserToken() : MathUtils.ParseJson <UserToken>(Encoding.UTF8.GetString(val));
}, new RedisPoolSetting()
{
Host = redisHost, DbIndex = redisDb
});
return(userToken);
}
/// <summary>
/// put error sql
/// </summary>
/// <param name="value"></param>
/// <param name="key"></param>
/// <returns></returns>
private static bool PutError(byte[] value, string key = null)
{
bool result = false;
try
{
RedisConnectionPool.Process(client => client.ZAdd(SlaveMessageQueue + (string.IsNullOrEmpty(key) ? SqlSyncErrorQueueKey : key), DateTime.Now.Ticks, value));
result = true;
}
catch (Exception ex)
{
TraceLog.WriteError("Sql error queue write error:{0}", ex);
}
return(result);
}
/// <summary>
/// put error sql
/// </summary>
/// <param name="value"></param>
/// <returns></returns>
private static bool PutError(byte[] value)
{
bool result = false;
try
{
RedisConnectionPool.Process(client => client.ZAdd(SqlSyncErrorQueueKey, DateTime.Now.Ticks, value));
result = true;
}
catch (Exception ex)
{
TraceLog.WriteError("Sql error queue write error:{0}", ex);
}
return(result);
}
/// <summary>
/// 放到Redsi中
/// </summary>
/// <param name="statement"></param>
public static bool Put(SqlStatement statement)
{
bool result = false;
try
{
string key = GetSqlQueueKey(statement.IdentityID);
byte[] value = ProtoBufUtils.Serialize(statement);
RedisConnectionPool.Process(client => client.ZAdd(key, DateTime.Now.Ticks, value));
result = true;
}
catch (Exception ex)
{
TraceLog.WriteError("Sql update queue write error:{0}\r\n{1}", ex, JsonUtils.SerializeCustom(statement));
}
return(result);
}
public void RedisTestConnect()
{
var date = DateTime.Now;
var tasks = new Task[1000];
for (int i = 0; i < tasks.Count(); i++)
{
tasks[i] = Task.Factory.StartNew(() =>
{
while (date.AddMinutes(1) > DateTime.Now)
{
RedisConnectionPool.Process(c => c.HLen("$GameServer.Model.GameUser"));
Thread.Sleep(100);
}
});
}
Task.WaitAll(tasks);
}
/// <summary>
///
/// </summary>
/// <param name="key"></param>
/// <param name="userToken"></param>
public static void SaveToken(string key, UserToken userToken)
{
string userKey = string.Format("{0}:{1}", AccountServerToken, userToken.UserId);
string redisKey = string.Format("{0}@{1}", AccountServerToken, key);
if (!string.IsNullOrEmpty(RedisHost) && RedisConnected)
{
//use redis cache.
RedisConnectionPool.Process(client =>
{
string oldToken = client.Get <string>(userKey);
if (!string.IsNullOrEmpty(oldToken))
{
client.Remove(string.Format("{0}@{1}", AccountServerToken, oldToken));
}
client.Set(redisKey, userToken, userToken.ExpireTime);
client.Set(userKey, key, userToken.ExpireTime);
});
return;
}
if (HttpContext.Current != null && HttpContext.Current.Cache != null)
{
var cache = HttpContext.Current.Cache;
var oldToken = cache[userKey] as string;
if (!string.IsNullOrEmpty(oldToken))
{
cache.Remove(string.Format("{0}@{1}", AccountServerToken, oldToken));
}
cache[redisKey] = userToken;
cache[userKey] = key;
}
else
{
string oldToken;
if (userHashCache.TryGetValue(userToken.UserId, out oldToken))
{
UserToken temp;
userTokenCache.TryRemove(oldToken, out temp);
}
userTokenCache[key] = userToken;
}
}
/// <summary>
/// Send to queue pool
/// </summary>
/// <param name="entityList"></param>
public static void Send(AbstractEntity[] entityList)
{
string key = "";
try
{
if (entityList == null || entityList.Length == 0)
{
return;
}
var groupList = entityList.GroupBy(t => t.GetIdentityId());
foreach (var g in groupList)
{
var valueList = g.ToList();
byte[][] keyBytes = new byte[valueList.Count][];
byte[][] valueBytes = new byte[valueList.Count][];
string queueKey = GetRedisSyncQueueKey(g.Key);
byte[] idBytes = BufferUtils.GetBytes(g.Key);
int index = 0;
foreach (var entity in valueList)
{
key = string.Format("{0}_{1}", entity.GetType().FullName, entity.GetKeyCode());
keyBytes[index] = RedisConnectionPool.ToByteKey(key);
byte[] stateBytes = BufferUtils.GetBytes(entity.IsDelete ? 1 : 0);
valueBytes[index] = BufferUtils.MergeBytes(
BufferUtils.GetBytes(idBytes.Length + stateBytes.Length),
idBytes,
stateBytes,
ProtoBufUtils.Serialize(entity));
index++;
}
RedisConnectionPool.Process(client => client.HMSet(queueKey, keyBytes, valueBytes));
}
}
catch (Exception ex)
{
TraceLog.WriteError("Post changed key:{0} error:{1}", key, ex);
}
}
/// <summary>
/// Process synchronized wait queue of Sql
/// </summary>
/// <param name="waitQueueKey"></param>
/// <param name="keys"></param>
/// <param name="values"></param>
private static void DoProcessSqlWaitSyncQueue(string waitQueueKey, byte[][] keys, byte[][] values)
{
try
{
var sqlSender = new SqlDataSender();
RedisConnectionPool.Process(client =>
{
var sqlList = GenerateSqlFrom(sqlSender, client, keys, values);
var groupSqlList = sqlList.GroupBy(t => t.Key);
foreach (var g in groupSqlList)
{
var pairs = g.Select(t => t.Value).ToList();
client.ZAdd(g.Key, pairs);
}
});
}
catch (Exception ex)
{
TraceLog.WriteError("DoProcessSqlWaitSyncQueue error:{0}", ex);
}
}
/// <summary>
///
/// </summary>
/// <param name="key"></param>
/// <returns></returns>
public static UserToken GetUserToken(string key)
{
UserToken userToken = null;
if (!string.IsNullOrEmpty(RedisHost) && RedisConnected)
{
//use redis cache.
RedisConnectionPool.Process(client =>
{
userToken = client.Get <UserToken>(string.Format("{0}@{1}", AccountServerToken, key));
});
return(userToken);
}
if (HttpContext.Current != null && HttpContext.Current.Cache != null)
{
userToken = (UserToken)HttpContext.Current.Cache[key];
}
if (Equals(userToken, null) && userTokenCache.ContainsKey(key))
{
userToken = (UserToken)userTokenCache[key];
}
return(userToken);
}
/// <summary>
/// Process synchronized queue of redis
/// </summary>
/// <param name="sysnWorkingQueueKey"></param>
/// <param name="keys"></param>
/// <param name="values"></param>
private static bool ProcessRedisSyncQueue(string sysnWorkingQueueKey, byte[][] keys, byte[][] values)
{
bool result = false;
try
{
var redisSyncErrorQueue = new List <byte[][]>();
var entityList = new List <RedisEntityItem>();
var entityRemList = new List <RedisEntityItem>();
var mutilKeyMapList = new List <RedisEntityItem>();
var mutilKeyMapRemList = new List <RedisEntityItem>();
var sqlWaitSyncQueue = new List <KeyValuePair <string, byte[][]> >();
for (int i = 0; i < keys.Length; i++)
{
byte[] keyBytes = keys[i];
byte[] valueBytes = values[i];
try
{
string[] queueKey = RedisConnectionPool.ToStringKey(keyBytes).Split('_');
string entityTypeName = RedisConnectionPool.DecodeTypeName(queueKey[0]);
string entityParentKey = RedisConnectionPool.GetRedisEntityKeyName(queueKey[0]);
byte[] entityKeyBytes = RedisConnectionPool.ToByteKey(queueKey[1]);
bool hasMutilKey = false;
bool isStoreInDb = true;
SchemaTable schema;
if (EntitySchemaSet.TryGet(entityTypeName, out schema))
{
hasMutilKey = RedisConnectionPool.CurrRedisInfo.ClientVersion >= RedisStorageVersion.HashMutilKeyMap &&
schema.EntityType.IsSubclassOf(typeof(BaseEntity)) &&
schema.Keys.Length > 1;
isStoreInDb = schema.StorageType.HasFlag(StorageType.WriteOnlyDB) || schema.StorageType.HasFlag(StorageType.ReadWriteDB);
}
byte[] headBytes;
byte[] entityValBytes;
int state;
int identity;
DecodeValueBytes(valueBytes, out headBytes, out entityValBytes, out state, out identity);
var entityItem = new RedisEntityItem()
{
HashId = entityParentKey,
UserId = identity,
KeyBytes = entityKeyBytes,
ValueBytes = entityValBytes,
State = state,
HasMutilKey = hasMutilKey
};
if (entityItem.State == 1)
{
entityRemList.Add(entityItem);
if (hasMutilKey)
{
mutilKeyMapRemList.Add(entityItem);
}
}
else
{
entityList.Add(entityItem);
if (hasMutilKey)
{
mutilKeyMapList.Add(entityItem);
}
}
if (_enableWriteToDb && isStoreInDb)
{
//增加到Sql等待队列
string sqlWaitQueueKey = GetSqlWaitSyncQueueKey(identity);
sqlWaitSyncQueue.Add(new KeyValuePair <string, byte[][]>(sqlWaitQueueKey, new[] { keyBytes, headBytes }));
}
}
catch
{
redisSyncErrorQueue.Add(new[] { keyBytes, valueBytes });
}
}
var redisErrorKeys = redisSyncErrorQueue.Select(p => p[0]).ToArray();
var redisErrorValues = redisSyncErrorQueue.Select(p => p[1]).ToArray();
var sqlWaitGroups = sqlWaitSyncQueue.GroupBy(p => p.Key);
var setGroups = entityList.GroupBy(p => p.HashId);
var removeGroups = entityRemList.GroupBy(p => p.HashId);
var mutilKeyMapGroups = mutilKeyMapList.GroupBy(p => p.HashId);
var mutilKeyMapRemGroups = mutilKeyMapRemList.GroupBy(p => p.HashId);
RedisConnectionPool.ProcessPipeline(pipeline =>
{
bool hasPost = false;
foreach (var g in setGroups)
{
var entityKeys = g.Select(p => p.KeyBytes).ToArray();
var entityValues = g.Select(p => p.ValueBytes).ToArray();
pipeline.QueueCommand(client => ((RedisClient)client).HMSet(g.Key, entityKeys, entityValues));
hasPost = true;
}
foreach (var g in removeGroups)
{
var keybytes = g.Select(p => p.KeyBytes).ToArray();
pipeline.QueueCommand(client => ((RedisClient)client).HDel(g.Key, keybytes));
hasPost = true;
}
foreach (var g in mutilKeyMapGroups)
{
string hashId = g.Key;
var subGroup = g.GroupBy(t => t.UserId);
foreach (var @group in subGroup)
{
string firstKey = AbstractEntity.EncodeKeyCode(@group.Key.ToString());
var keybytes = @group.Select(p => p.KeyBytes).ToArray();
pipeline.QueueCommand(client => RedisConnectionPool.SetMutilKeyMap((RedisClient)client, hashId, firstKey, keybytes));
hasPost = true;
}
}
foreach (var g in mutilKeyMapRemGroups)
{
string hashId = g.Key;
var subGroup = g.GroupBy(t => t.UserId);
foreach (var @group in subGroup)
{
string firstKey = AbstractEntity.EncodeKeyCode(@group.Key.ToString());
var keybytes = @group.Select(p => p.KeyBytes).ToArray();
pipeline.QueueCommand(client => RedisConnectionPool.RemoveMutilKeyMap((RedisClient)client, hashId, firstKey, keybytes));
hasPost = true;
}
}
if (redisErrorKeys.Length > 0)
{
pipeline.QueueCommand(client => ((RedisClient)client).HMSet(RedisSyncErrorQueueKey, redisErrorKeys, redisErrorValues));
hasPost = true;
}
foreach (var g in sqlWaitGroups)
{
var sqlWaitKeys = g.Select(p => p.Value[0]).ToArray();
var sqlWaitValues = g.Select(p => p.Value[1]).ToArray();
pipeline.QueueCommand(client => ((RedisClient)client).HMSet(g.Key, sqlWaitKeys, sqlWaitValues));
hasPost = true;
}
if (hasPost)
{
pipeline.Flush();
}
result = redisErrorKeys.Length == 0;
});
}
catch (Exception ex)
{
TraceLog.WriteError("DoProcessRedisSyncQueue error:{0}", ex);
try
{
RedisConnectionPool.Process(client => client.HMSet(RedisSyncErrorQueueKey, keys, values));
}
catch (Exception er)
{
TraceLog.WriteError("Put RedisSyncErrorQueue error:{0}", er);
}
}
return(result);
}
/// <summary>
/// Check to be synchronized wait queue of Sql
/// </summary>
/// <param name="state"></param>
private static void OnCheckSqlWaitSyncQueue(object state)
{
int identity = (int)state;
if (Interlocked.CompareExchange(ref _isSqlWaitSyncWorking[identity], 1, 0) == 0)
{
try
{
string queueKey = GetSqlWaitSyncQueueKey(identity);
string workingKey = queueKey + "_temp";
byte[][] keys = null;
byte[][] values = null;
RedisConnectionPool.ProcessReadOnly(client =>
{
bool hasWorkingQueue = client.HLen(workingKey) > 0;
bool hasNewWorkingQueue = client.HLen(queueKey) > 0;
if (!hasWorkingQueue && !hasNewWorkingQueue)
{
return;
}
if (!hasWorkingQueue)
{
try
{
client.Rename(queueKey, workingKey);
}
catch
{
}
}
var keyValuePairs = client.HGetAll(workingKey);
if (keyValuePairs != null && keyValuePairs.Length > 0)
{
keys = keyValuePairs.Where((buffs, index) => index % 2 == 0).ToArray();
values = keyValuePairs.Where((buffs, index) => index % 2 == 1).ToArray();
client.Remove(workingKey);
}
});
if (keys != null && keys.Length > 0)
{
try
{
int pos = 0;
byte[][] subKeys, subValues;
while (true)
{
int count = pos + sqlWaitPackSize < keys.Length ? sqlWaitPackSize : keys.Length - pos;
if (count <= 0)
{
break;
}
subKeys = new byte[count][];
subValues = new byte[count][];
Array.Copy(keys, pos, subKeys, 0, subKeys.Length);
Array.Copy(values, pos, subValues, 0, subValues.Length);
_threadPools.QueueWorkItem(DoProcessSqlWaitSyncQueue, workingKey, subKeys, subValues);
pos += sqlWaitPackSize;
}
}
catch (Exception er)
{
TraceLog.WriteError("OnCheckSqlWaitSyncQueue error:{0}", er);
RedisConnectionPool.Process(client => client.HMSet(SqlSyncWaitErrirQueueKey, keys, values));
}
}
}
catch (Exception ex)
{
TraceLog.WriteError("OnCheckSqlWaitSyncQueue error:{0}", ex);
}
finally
{
Interlocked.Exchange(ref _isSqlWaitSyncWorking[identity], 0);
}
}
}
/// <summary>
/// Process synchronized queue of redis
/// </summary>
/// <param name="sysnWorkingQueueKey"></param>
/// <param name="keys"></param>
/// <param name="values"></param>
private static void DoProcessRedisSyncQueue(string sysnWorkingQueueKey, byte[][] keys, byte[][] values)
{
try
{
var redisSyncErrorQueue = new List <byte[][]>();
var setList = new List <KeyValuePair <string, byte[][]> >();
var removeList = new List <KeyValuePair <string, byte[]> >();
var sqlWaitSyncQueue = new List <KeyValuePair <string, byte[][]> >();
for (int i = 0; i < keys.Length; i++)
{
byte[] keyBytes = keys[i];
byte[] valueBytes = values[i];
try
{
string[] queueKey = RedisConnectionPool.ToStringKey(keyBytes).Split('_');
string entityTypeName = RedisConnectionPool.DecodeTypeName(queueKey[0]);
string entityParentKey = RedisConnectionPool.GetRedisEntityKeyName(queueKey[0]);
byte[] entityKeyBytes = RedisConnectionPool.ToByteKey(queueKey[1]);
byte[] headBytes;
byte[] entityValBytes;
int state;
int identity;
DecodeValueBytes(valueBytes, out headBytes, out entityValBytes, out state, out identity);
if (state == 1)
{
removeList.Add(new KeyValuePair <string, byte[]>(entityParentKey, entityKeyBytes));
}
else
{
setList.Add(new KeyValuePair <string, byte[][]>(entityParentKey, new[] { entityKeyBytes, entityValBytes }));
}
bool isStoreInDb = true;
SchemaTable schema;
if (EntitySchemaSet.TryGet(entityTypeName, out schema))
{
isStoreInDb = schema.StorageType.HasFlag(StorageType.WriteOnlyDB) || schema.StorageType.HasFlag(StorageType.ReadWriteDB);
}
if (_enableWriteToDb && isStoreInDb)
{
//增加到Sql等待队列
string sqlWaitQueueKey = GetSqlWaitSyncQueueKey(identity);
sqlWaitSyncQueue.Add(new KeyValuePair <string, byte[][]>(sqlWaitQueueKey, new[] { keyBytes, headBytes }));
}
}
catch
{
redisSyncErrorQueue.Add(new[] { keyBytes, valueBytes });
}
}
var redisErrorKeys = redisSyncErrorQueue.Select(p => p[0]).ToArray();
var redisErrorValues = redisSyncErrorQueue.Select(p => p[1]).ToArray();
var sqlWaitGroups = sqlWaitSyncQueue.GroupBy(p => p.Key);
var setGroups = setList.GroupBy(p => p.Key);
var removeGroups = removeList.GroupBy(p => p.Key);
RedisConnectionPool.Process(client =>
{
foreach (var g in setGroups)
{
var entityKeys = g.Select(p => p.Value[0]).ToArray();
var entityValues = g.Select(p => p.Value[1]).ToArray();
client.HMSet(g.Key, entityKeys, entityValues);
}
foreach (var g in removeGroups)
{
var keybytes = g.Select(p => p.Value).ToArray();
client.HDel(g.Key, keybytes);
}
if (redisErrorKeys.Length > 0)
{
client.HMSet(RedisSyncErrorQueueKey, redisErrorKeys, redisErrorValues);
}
foreach (var g in sqlWaitGroups)
{
var sqlWaitKeys = g.Select(p => p.Value[0]).ToArray();
var sqlWaitValues = g.Select(p => p.Value[1]).ToArray();
client.HMSet(g.Key, sqlWaitKeys, sqlWaitValues);
}
});
}
catch (Exception ex)
{
TraceLog.WriteError("DoProcessRedisSyncQueue error:{0}", ex);
try
{
RedisConnectionPool.Process(client => client.HMSet(RedisSyncErrorQueueKey, keys, values));
}
catch (Exception er)
{
TraceLog.WriteError("Put RedisSyncErrorQueue error:{0}", er);
}
}
}
private static void OnEntitySyncQueue(object state)
{
if (Interlocked.CompareExchange(ref _entityQueueRunning, 1, 0) == 0)
{
try
{
var tempRemove = Interlocked.Exchange(ref _entityRemoteSet, new HashSet <string>());
var temp = Interlocked.Exchange(ref _entitySet, new HashSet <string>());
if (temp.Count == 0 || _queueWatchTimers == null)
{
return;
}
TraceLog.WriteWarn("OnEntitySyncQueue execute count:{0}, success:{1}/total {2}, fail:{3} start...", temp.Count, ExecuteSuccessCount, SendWaitCount, ExecuteFailCount);
RedisConnectionPool.Process(client =>
{
while (temp.Count > 0)
{
var dumpSet = temp.Take(ExecuteCount).ToArray();
var pipeline = client.CreatePipeline();
try
{
bool hasPost = false;
foreach (var key in dumpSet)
{
var keyValues = key.Split('_', '|');
if (keyValues.Length != 3)
{
TraceLog.WriteWarn("OnEntitySyncQueue:{0}", key);
ExecuteFailCount++;
continue;
}
AbstractEntity entity = CacheFactory.GetPersonalEntity(key) as AbstractEntity;
int id = keyValues[1].ToInt();
string keyCode = keyValues[2];
string redisKey = string.Format("{0}_{1}", keyValues[0], keyCode);
string hashId = GetRedisSyncQueueKey(id);
byte[] idBytes = BufferUtils.GetBytes(id);
var keyBytes = RedisConnectionPool.ToByteKey(redisKey);
bool isDelete;
byte[] entityBytes;
if (entity != null)
{
isDelete = entity.IsDelete;
entityBytes = _serializer.Serialize(entity);
//modify resean: set unchange status.
entity.Reset();
}
else if (tempRemove.Contains(key))
{
entityBytes = new byte[0];
isDelete = true;
}
else
{
ExecuteFailCount++;
TraceLog.WriteError("EntitySync queue key {0} faild object is null.", key);
continue;
}
byte[] stateBytes = BufferUtils.GetBytes(isDelete ? 1 : 0);
byte[] values =
BufferUtils.MergeBytes(BufferUtils.GetBytes(idBytes.Length + stateBytes.Length),
idBytes, stateBytes, entityBytes);
pipeline.QueueCommand(c =>
{
((RedisClient)c).HSet(hashId, keyBytes, values);
});
hasPost = true;
ExecuteSuccessCount++;
}
if (hasPost)
{
pipeline.Flush();
}
}
finally
{
pipeline.Dispose();
}
try
{
foreach (var key in dumpSet)
{
temp.Remove(key);
}
}
catch (Exception)
{
}
}
});
}
catch (Exception ex)
{
TraceLog.WriteError("EntitySync queue {0}", ex);
}
finally
{
Interlocked.Exchange(ref _entityQueueRunning, 0);
}
}
}
/// <summary>
/// 从历史库中加载数据
/// </summary>
/// <typeparam name="T"></typeparam>
/// <param name="redisKey"></param>
/// <param name="dataList"></param>
/// <returns></returns>
public bool TryLoadHistory <T>(string redisKey, out List <T> dataList) where T : ISqlEntity
{
string[] entityAndKeys = (redisKey ?? "").Split('_');
var entityKey = entityAndKeys.Length > 1 ? entityAndKeys[1] : null;
string entityNameKey = RedisConnectionPool.GetRedisEntityKeyName(entityAndKeys[0]);
bool result = false;
dataList = null;
SchemaTable schemaTable;
if (EntitySchemaSet.TryGet <EntityHistory>(out schemaTable))
{
try
{
var provider = DbConnectionProvider.CreateDbProvider(schemaTable);
if (provider == null)
{
//DB is optional and can no DB configuration
dataList = new List <T>();
return(true);
}
TransReceiveParam receiveParam = new TransReceiveParam(entityNameKey);
receiveParam.Schema = schemaTable;
int maxCount = receiveParam.Schema.Capacity;
var filter = new DbDataFilter(maxCount);
string key = schemaTable.Keys[0];
var entitySchema = EntitySchemaSet.Get(entityAndKeys[0]);
if (entitySchema != null && entitySchema.Keys.Length == 1)
{
filter.Condition = provider.FormatFilterParam(key);
filter.Parameters.Add(key, string.Format("{0}_{1}", entityNameKey, entityKey));
}
else
{
filter.Condition = provider.FormatFilterParam(key, "LIKE");
filter.Parameters.Add(key, string.Format("{0}_%{1}%", entityNameKey, entityKey));
}
receiveParam.DbFilter = filter;
List <EntityHistory> historyList;
if (_dbTransponder.TryReceiveData(receiveParam, out historyList))
{
if (historyList.Count == 0)
{
dataList = new List <T>();
return(true);
}
dataList = new List <T>();
var keyBytes = new byte[historyList.Count][];
var valueBytes = new byte[historyList.Count][];
for (int i = 0; i < keyBytes.Length; i++)
{
var entityHistory = historyList[i];
keyBytes[i] = RedisConnectionPool.ToByteKey(entityHistory.Key.Split('_')[1]);
valueBytes[i] = entityHistory.Value;
dataList.Add((T)_serializer.Deserialize(entityHistory.Value, typeof(T)));
}
//从DB备份中恢复到Redis, 多个Key时也要更新
var entitys = dataList.ToArray();
RedisConnectionPool.Process(client =>
{
client.HMSet(entityNameKey, keyBytes, valueBytes);
if (entitySchema.Keys.Length > 1)
{
RedisConnectionPool.UpdateFromMutilKeyMap <T>(client, entityKey, entitys);
}
});
result = true;
}
}
catch (Exception ex)
{
TraceLog.WriteError("Try load Redis's history key:{0}\r\nerror:{1}", entityNameKey, ex);
}
}
else
{
dataList = new List <T>();
result = true;
}
return(result);
}
/// <summary>
/// 通过Redis键获取实体对象
/// </summary>
/// <param name="key"></param>
/// <param name="isRemove"></param>
/// <param name="type"></param>
/// <param name="serializer"></param>
/// <returns></returns>
public static dynamic GetEntityFromRedis(string key, bool isRemove, Type type, ICacheSerializer serializer)
{
string typeName;
string asmName;
bool isEntityType;
string redisKey;
string entityKey = GetEntityTypeFromKey(key, out typeName, ref type, out asmName, out isEntityType, out redisKey);
dynamic entity = null;
RedisConnectionPool.Process(client =>
{
if (isEntityType)
{
var data = client.Get <byte[]>(redisKey);
if (data != null && type != null)
{
entity = serializer.Deserialize(data, type);
}
}
else
{
var data = client.Get <byte[]>(redisKey);
if (data != null && type != null)
{
var dict = (IDictionary)serializer.Deserialize(data, type);
entity = dict[entityKey];
}
}
if (entity == null)
{
//新版本Hash格式
var data = client.HGet(typeName, RedisConnectionPool.ToByteKey(entityKey));
if (data != null && type != null)
{
entity = serializer.Deserialize(data, type);
}
}
if (isRemove && entity == null && type != null)
{
//临时队列删除Entity
string setId = (isEntityType ? RedisConnectionPool.EncodeTypeName(typeName) : redisKey) + ":remove";
IDictionary dict = null;
RedisConnectionPool.ProcessTrans(client, new string[] { setId }, () =>
{
var data = client.Get <byte[]>(setId);
if (data == null)
{
return(false);
}
dict = (IDictionary)serializer.Deserialize(data, type);
entity = dict[entityKey];
dict.Remove(entityKey);
return(true);
}, trans =>
{
if (dict != null && dict.Count > 0)
{
trans.QueueCommand(c => c.Set(setId, serializer.Serialize(dict)));
}
else
{
trans.QueueCommand(c => c.Remove(setId));
}
}, null);
}
});
return(entity);
}
public bool TryLoadHistory <T>(string redisKey, out List <T> dataList)
{
redisKey = RedisConnectionPool.GetRedisEntityKeyName(redisKey);
bool result = false;
dataList = null;
SchemaTable schemaTable;
if (EntitySchemaSet.TryGet <EntityHistory>(out schemaTable))
{
try
{
var provider = DbConnectionProvider.CreateDbProvider(schemaTable);
if (provider == null)
{
//DB is optional and can no DB configuration
dataList = new List <T>();
return(true);
}
TransReceiveParam receiveParam = new TransReceiveParam(redisKey);
receiveParam.Schema = schemaTable;
int maxCount = receiveParam.Schema.Capacity;
var filter = new DbDataFilter(maxCount);
string key = schemaTable.Keys[0];
filter.Condition = provider.FormatFilterParam(key);
filter.Parameters.Add(key, redisKey);
receiveParam.DbFilter = filter;
receiveParam.Capacity = maxCount;
List <EntityHistory> historyList;
if (_dbTransponder.TryReceiveData(receiveParam, out historyList))
{
EntityHistory history = historyList.Count > 0 ? historyList[0] : null;
if (history != null && history.Value != null && history.Value.Length > 0)
{
byte[][] bufferBytes = ProtoBufUtils.Deserialize <byte[][]>(history.Value);
byte[][] keys = bufferBytes.Where((b, index) => index % 2 == 0).ToArray();
byte[][] values = bufferBytes.Where((b, index) => index % 2 == 1).ToArray();
RedisConnectionPool.Process(client => client.HMSet(redisKey, keys, values));
dataList = values.Select(value => ProtoBufUtils.Deserialize <T>(value)).ToList();
result = true;
}
else
{
dataList = new List <T>();
result = true;
}
}
}
catch (Exception ex)
{
TraceLog.WriteError("Try load Redis's history key:{0}\r\nerror:{1}", redisKey, ex);
}
}
else
{
dataList = new List <T>();
result = true;
}
return(result);
}
