private static string GetEntityTypeFromKey(string key, out string typeName, ref Type type, out string asmName, out bool isEntityType, out string redisKey)
{
int index = key.IndexOf(',');
var arr = (index > -1 ? key.Substring(0, index) : key).Split('_');
typeName = arr[0];
asmName = index == -1 ? "" : key.Substring(index + 1, key.Length - index - 1);
string persionKey = string.Empty;
string entityKey = string.Empty;
if (arr.Length > 1)
{
entityKey = arr[1];
var tempArr = entityKey.Split('|');
if (tempArr.Length > 1)
{
persionKey = tempArr[0];
entityKey = tempArr[1];
}
}
isEntityType = false;
if (string.IsNullOrEmpty(persionKey))
{
isEntityType = true;
redisKey = string.Format("{0}_{1}", RedisConnectionPool.EncodeTypeName(typeName), entityKey);
}
else
{
//私有类型
redisKey = string.Format("{0}_{1}", RedisConnectionPool.EncodeTypeName(typeName), persionKey);
}
string formatString = entityTypeNameFormat;
if (isEntityType)
{
formatString = "{0},{1}";
}
if (type == null)
{
string entityTypeName = RedisConnectionPool.DecodeTypeName(typeName);
type = Type.GetType(string.Format(formatString, entityTypeName, asmName), false, true);
if (Equals(type, null))
{
var enitityAsm = ScriptEngines.GetEntityAssembly();
if (enitityAsm != null)
{
asmName = enitityAsm.GetName().Name;
type = Type.GetType(string.Format(formatString, entityTypeName, asmName), false, true);
if (Equals(type, null))
{
//调试模式下type为空处理
type = enitityAsm.GetType(entityTypeName, false, true);
}
}
}
}
return(entityKey);
}
/// <summary>
/// 获取Container中存在的Cache，不重新创建
/// </summary>
/// <param name="redisKey"></param>
/// <param name="itemPair"></param>
/// <param name="periodTime"></param>
/// <returns></returns>
public static bool TryGetContainerCacheItem(string redisKey, out KeyValuePair <string, CacheItemSet> itemPair, int periodTime = 0)
{
itemPair = default(KeyValuePair <string, CacheItemSet>);
CacheItemSet cacheItem;
string[] keys = (redisKey ?? "").Split('_');
if (keys.Length == 2 && !string.IsNullOrEmpty(keys[0]))
{
CacheContainer container = null;
string typeName = RedisConnectionPool.DecodeTypeName(keys[0]);
var schema = EntitySchemaSet.Get(typeName);
periodTime = periodTime > 0 ? periodTime : schema.PeriodTime;
if (_writePools != null && !_writePools.TryGetValue(typeName, out container))
{
_writePools.InitContainer(typeName);
_writePools.TryGetValue(typeName, out container);
}
if (container == null)
{
return(false);
}
string[] childKeys = keys[1].Split('|');
string personalKey = childKeys[0];
string entityKey = childKeys.Length > 1 ? childKeys[1] : "";
if (schema.CacheType == CacheType.Dictionary)//|| schema.CacheType == CacheType.Entity)
{
var result = container.Collection.TryGetValue(personalKey, out cacheItem);
itemPair = new KeyValuePair <string, CacheItemSet>(entityKey, cacheItem);
return(result);
}
if (schema.CacheType == CacheType.Entity)
{
var result = container.Collection.TryGetValue(entityKey, out cacheItem);
itemPair = new KeyValuePair <string, CacheItemSet>(entityKey, cacheItem);
return(result);
}
if (schema.CacheType == CacheType.Queue)
{
TraceLog.WriteError("Not support CacheType.Queue get cache, key:{0}.", redisKey);
}
////存在分类id与实体主键相同情况, 要优先判断实体主键
//if (!string.IsNullOrEmpty(personalKey) && container.Collection.TryGetValue(entityKey, out cacheItem))
//{
// itemPair = new KeyValuePair<string, CacheItemSet>(entityKey, cacheItem);
// return true;
//}
//if (!string.IsNullOrEmpty(personalKey) && container.Collection.TryGetValue(personalKey, out cacheItem))
//{
// itemPair = new KeyValuePair<string, CacheItemSet>(entityKey, cacheItem);
// return true;
//}
}
return(false);
}
/// <summary>
/// 通过Redis键从缓存中获取实体对象
/// </summary>
/// <param name="redisKey"></param>
/// <param name="itemSet"></param>
/// <returns></returns>
public static dynamic GetPersonalEntity(string redisKey, out CacheItemSet itemSet)
{
itemSet = null;
dynamic entity = null;
string[] keys = (redisKey ?? "").Split('_');
if (keys.Length == 2 && !string.IsNullOrEmpty(keys[0]))
{
CacheContainer container;
string typeName = RedisConnectionPool.DecodeTypeName(keys[0]);
if (_writePools != null && _writePools.TryGetValue(typeName, out container))
{
string[] childKeys = keys[1].Split('|');
string personalKey = childKeys[0];
string entityKey = childKeys.Length > 1 ? childKeys[1] : "";
if (!string.IsNullOrEmpty(personalKey) &&
(container.Collection.TryGetValue(entityKey, out itemSet) ||
container.Collection.TryGetValue(personalKey, out itemSet)))
{
switch (itemSet.ItemType)
{
case CacheType.Entity:
entity = itemSet.ItemData;
break;
case CacheType.Dictionary:
var set = itemSet.ItemData as BaseCollection;
if (set != null)
{
set.TryGetValue(entityKey, out entity);
}
break;
default:
TraceLog.WriteError("Not suported CacheType:{0} for GetPersonalEntity key:{1}", itemSet.ItemType, redisKey);
break;
}
}
}
}
if (entity == null)
{
//while is remove entity is empty.
//TraceLog.WriteComplement("GetPersonalEntity key:{0} is empty.", redisKey);
}
return(entity);
}
/// <summary>
///
/// </summary>
/// <param name="redisKey"></param>
/// <param name="itemPair">key:entity's key, value:</param>
/// <param name="periodTime"></param>
/// <returns></returns>
public static bool TryGetCacheItem(string redisKey, out KeyValuePair <string, CacheItemSet> itemPair, int periodTime = 0)
{
itemPair = default(KeyValuePair <string, CacheItemSet>);
CacheItemSet cacheItem;
string[] keys = (redisKey ?? "").Split('_');
if (keys.Length == 2 && !string.IsNullOrEmpty(keys[0]))
{
CacheContainer container = null;
string typeName = RedisConnectionPool.DecodeTypeName(keys[0]);
var schema = EntitySchemaSet.Get(typeName);
periodTime = periodTime > 0 ? periodTime : schema.PeriodTime;
if (_writePools != null && !_writePools.TryGetValue(typeName, out container))
{
_writePools.InitContainer(typeName);
_writePools.TryGetValue(typeName, out container);
}
if (container == null)
{
return(false);
}
string[] childKeys = keys[1].Split('|');
string personalKey = childKeys[0];
string entityKey = childKeys.Length > 1 ? childKeys[1] : "";
if (schema.CacheType == CacheType.Dictionary)
{
var lazy = new Lazy <CacheItemSet>(() =>
{
bool isReadonly = schema.AccessLevel == AccessLevel.ReadOnly;
BaseCollection itemCollection = isReadonly
? (BaseCollection) new ReadonlyCacheCollection()
: new CacheCollection(schema.IsMutilKey ? 0 : 1);
var itemSet = new CacheItemSet(schema.CacheType, periodTime, isReadonly);
if (!isReadonly && _writePools.Setting != null)
{
itemSet.OnChangedNotify += _writePools.Setting.OnChangedNotify;
}
itemSet.HasCollection = true;
itemSet.SetItem(itemCollection);
return(itemSet);
});
cacheItem = container.Collection.GetOrAdd(personalKey, key => lazy.Value);
itemPair = new KeyValuePair <string, CacheItemSet>(entityKey, cacheItem);
return(true);
}
if (schema.CacheType == CacheType.Entity)
{
var lazy = new Lazy <CacheItemSet>(() =>
{
bool isReadonly = schema.AccessLevel == AccessLevel.ReadOnly;
var itemSet = new CacheItemSet(schema.CacheType, periodTime, isReadonly);
if (!isReadonly && _writePools.Setting != null)
{
itemSet.OnChangedNotify += _writePools.Setting.OnChangedNotify;
}
return(itemSet);
});
cacheItem = container.Collection.GetOrAdd(entityKey, key => lazy.Value);
itemPair = new KeyValuePair <string, CacheItemSet>(entityKey, cacheItem);
return(true);
}
if (schema.CacheType == CacheType.Queue)
{
TraceLog.WriteError("Not support CacheType.Queue get cache, key:{0}.", redisKey);
}
////存在分类id与实体主键相同情况, 要优先判断实体主键
//if (!string.IsNullOrEmpty(personalKey) && container.Collection.TryGetValue(entityKey, out cacheItem))
//{
// itemPair = new KeyValuePair<string, CacheItemSet>(entityKey, cacheItem);
// return true;
//}
//if (!string.IsNullOrEmpty(personalKey) && container.Collection.TryGetValue(personalKey, out cacheItem))
//{
// itemPair = new KeyValuePair<string, CacheItemSet>(entityKey, cacheItem);
// return true;
//}
}
return(false);
}
/// <summary>
/// Get schema for typename
/// </summary>
/// <param name="typeName"></param>
/// <param name="schema"></param>
/// <returns></returns>
public static bool TryGet(string typeName, out SchemaTable schema)
{
typeName = RedisConnectionPool.DecodeTypeName(typeName);
return(SchemaSet.TryGetValue(typeName, out schema));
}
/// <summary>
/// Generate Sql statements from the Keys-Values
/// </summary>
/// <param name="sender"></param>
/// <param name="client"></param>
/// <param name="keys"></param>
/// <param name="values"></param>
/// <returns></returns>
private static IEnumerable <KeyValuePair <string, KeyValuePair <byte[], long> > > GenerateSqlFrom(SqlDataSender sender, RedisClient client, byte[][] keys, byte[][] values)
{
var typeKeyValuePairs = new List <KeyValuePair <string, byte[][]> >();
for (int i = 0; i < keys.Length; i++)
{
byte[] keyBytes = keys[i];
byte[] headBytes = values[i];
string entityTypeKey = RedisConnectionPool.ToStringKey(keyBytes);
string[] entityKeys = entityTypeKey.Split(',')[0].Split('_');
string typeName = entityKeys[0];
byte[] entityKeyBytes = RedisConnectionPool.ToByteKey(entityKeys[1]);
typeKeyValuePairs.Add(new KeyValuePair <string, byte[][]>(typeName, new[] { entityKeyBytes, keyBytes, headBytes })
);
}
var sqlList = new List <KeyValuePair <string, KeyValuePair <byte[], long> > >();
var typeGroup = typeKeyValuePairs.GroupBy(p => p.Key);
foreach (var g in typeGroup)
{
var typeName = g.Key;
try
{
string entityParentKey = RedisConnectionPool.GetRedisEntityKeyName(typeName);
string asmName = "";
var enitityAsm = EntitySchemaSet.EntityAssembly;
if (enitityAsm != null)
{
asmName = "," + enitityAsm.GetName().Name;
}
Type type = Type.GetType(string.Format("{0}{1}", RedisConnectionPool.DecodeTypeName(typeName), asmName), false, true);
if (type == null)
{
//调试模式下type为空处理
type = enitityAsm != null?enitityAsm.GetType(RedisConnectionPool.DecodeTypeName(typeName), false, true) : null;
if (type == null)
{
throw new ArgumentTypeException(string.Format("Get entity \"{0}\" type is null", entityParentKey));
}
}
var keyBuffers = g.Select(p => p.Value[0]).ToArray();
var headBuffers = g.Select(p => p.Value[2]).ToArray();
var valueBuffers = client.HMGet(entityParentKey, keyBuffers);
for (int i = 0; i < keyBuffers.Length; i++)
{
string keyCode = RedisConnectionPool.ToStringKey(keyBuffers[i]);
var buffer = valueBuffers != null && valueBuffers.Length > i ? valueBuffers[i] : null;
var headBytes = headBuffers[i];
int identity;
int state;
DecodeHeadBytes(headBytes, out identity, out state);
AbstractEntity entity = null;
if (state == 1 && buffer == null)
{
//entity remove ops
entity = type.CreateInstance(new object[0]) as AbstractEntity;
entity.IsDelete = true;
entity.SetKeyValue(keyCode, typeName);
}
else if (buffer != null)
{
entity = _serializer.Deserialize(buffer, type) as AbstractEntity;
}
if (entity != null)
{
if (state == 1)
{
entity.IsDelete = true;
}
SqlStatement statement = sender.GenerateSqlQueue(entity);
if (statement == null)
{
throw new Exception(string.Format("Generate sql of \"{0}\" entity error", typeName));
}
var sqlValueBytes = ProtoBufUtils.Serialize(statement);
string sqlQueueKey = SqlStatementManager.GetSqlQueueKey(statement.IdentityID);
sqlList.Add(new KeyValuePair <string, KeyValuePair <byte[], long> >(sqlQueueKey,
new KeyValuePair <byte[], long>(sqlValueBytes, DateTime.Now.Ticks)));
}
else
{
throw new Exception(string.Format("Get \"{0}\" entity is null, keycode:{1},buffer len:{2},state:{3}",
typeName, keyCode, buffer == null ? -1 : buffer.Length, state));
}
}
}
catch (Exception er)
{
TraceLog.WriteError("FindEntityFromRedis {0} error:{1}", typeName, er);
var errorKeys = g.Select(p => p.Value[1]).ToArray();
var errorValues = g.Select(p => p.Value[2]).ToArray();
client.HMSet(SqlSyncWaitErrirQueueKey, errorKeys, errorValues);
}
}
return(sqlList);
}
/// <summary>
/// Process synchronized queue of redis
/// </summary>
/// <param name="sysnWorkingQueueKey"></param>
/// <param name="keys"></param>
/// <param name="values"></param>
private static void DoProcessRedisSyncQueue(string sysnWorkingQueueKey, byte[][] keys, byte[][] values)
{
try
{
var redisSyncErrorQueue = new List <byte[][]>();
var setList = new List <KeyValuePair <string, byte[][]> >();
var removeList = new List <KeyValuePair <string, byte[]> >();
var sqlWaitSyncQueue = new List <KeyValuePair <string, byte[][]> >();
for (int i = 0; i < keys.Length; i++)
{
byte[] keyBytes = keys[i];
byte[] valueBytes = values[i];
try
{
string[] queueKey = RedisConnectionPool.ToStringKey(keyBytes).Split('_');
string entityTypeName = RedisConnectionPool.DecodeTypeName(queueKey[0]);
string entityParentKey = RedisConnectionPool.GetRedisEntityKeyName(queueKey[0]);
byte[] entityKeyBytes = RedisConnectionPool.ToByteKey(queueKey[1]);
byte[] headBytes;
byte[] entityValBytes;
int state;
int identity;
DecodeValueBytes(valueBytes, out headBytes, out entityValBytes, out state, out identity);
if (state == 1)
{
removeList.Add(new KeyValuePair <string, byte[]>(entityParentKey, entityKeyBytes));
}
else
{
setList.Add(new KeyValuePair <string, byte[][]>(entityParentKey, new[] { entityKeyBytes, entityValBytes }));
}
bool isStoreInDb = true;
SchemaTable schema;
if (EntitySchemaSet.TryGet(entityTypeName, out schema))
{
isStoreInDb = schema.StorageType.HasFlag(StorageType.WriteOnlyDB) || schema.StorageType.HasFlag(StorageType.ReadWriteDB);
}
if (_enableWriteToDb && isStoreInDb)
{
//增加到Sql等待队列
string sqlWaitQueueKey = GetSqlWaitSyncQueueKey(identity);
sqlWaitSyncQueue.Add(new KeyValuePair <string, byte[][]>(sqlWaitQueueKey, new[] { keyBytes, headBytes }));
}
}
catch
{
redisSyncErrorQueue.Add(new[] { keyBytes, valueBytes });
}
}
var redisErrorKeys = redisSyncErrorQueue.Select(p => p[0]).ToArray();
var redisErrorValues = redisSyncErrorQueue.Select(p => p[1]).ToArray();
var sqlWaitGroups = sqlWaitSyncQueue.GroupBy(p => p.Key);
var setGroups = setList.GroupBy(p => p.Key);
var removeGroups = removeList.GroupBy(p => p.Key);
RedisConnectionPool.Process(client =>
{
foreach (var g in setGroups)
{
var entityKeys = g.Select(p => p.Value[0]).ToArray();
var entityValues = g.Select(p => p.Value[1]).ToArray();
client.HMSet(g.Key, entityKeys, entityValues);
}
foreach (var g in removeGroups)
{
var keybytes = g.Select(p => p.Value).ToArray();
client.HDel(g.Key, keybytes);
}
if (redisErrorKeys.Length > 0)
{
client.HMSet(RedisSyncErrorQueueKey, redisErrorKeys, redisErrorValues);
}
foreach (var g in sqlWaitGroups)
{
var sqlWaitKeys = g.Select(p => p.Value[0]).ToArray();
var sqlWaitValues = g.Select(p => p.Value[1]).ToArray();
client.HMSet(g.Key, sqlWaitKeys, sqlWaitValues);
}
});
}
catch (Exception ex)
{
TraceLog.WriteError("DoProcessRedisSyncQueue error:{0}", ex);
try
{
RedisConnectionPool.Process(client => client.HMSet(RedisSyncErrorQueueKey, keys, values));
}
catch (Exception er)
{
TraceLog.WriteError("Put RedisSyncErrorQueue error:{0}", er);
}
}
}
/// <summary>
/// Process synchronized queue of redis
/// </summary>
/// <param name="sysnWorkingQueueKey"></param>
/// <param name="keys"></param>
/// <param name="values"></param>
private static bool ProcessRedisSyncQueue(string sysnWorkingQueueKey, byte[][] keys, byte[][] values)
{
bool result = false;
try
{
var redisSyncErrorQueue = new List <byte[][]>();
var entityList = new List <RedisEntityItem>();
var entityRemList = new List <RedisEntityItem>();
var mutilKeyMapList = new List <RedisEntityItem>();
var mutilKeyMapRemList = new List <RedisEntityItem>();
var sqlWaitSyncQueue = new List <KeyValuePair <string, byte[][]> >();
for (int i = 0; i < keys.Length; i++)
{
byte[] keyBytes = keys[i];
byte[] valueBytes = values[i];
try
{
string[] queueKey = RedisConnectionPool.ToStringKey(keyBytes).Split('_');
string entityTypeName = RedisConnectionPool.DecodeTypeName(queueKey[0]);
string entityParentKey = RedisConnectionPool.GetRedisEntityKeyName(queueKey[0]);
byte[] entityKeyBytes = RedisConnectionPool.ToByteKey(queueKey[1]);
bool hasMutilKey = false;
bool isStoreInDb = true;
SchemaTable schema;
if (EntitySchemaSet.TryGet(entityTypeName, out schema))
{
hasMutilKey = RedisConnectionPool.CurrRedisInfo.ClientVersion >= RedisStorageVersion.HashMutilKeyMap &&
schema.EntityType.IsSubclassOf(typeof(BaseEntity)) &&
schema.Keys.Length > 1;
isStoreInDb = schema.StorageType.HasFlag(StorageType.WriteOnlyDB) || schema.StorageType.HasFlag(StorageType.ReadWriteDB);
}
byte[] headBytes;
byte[] entityValBytes;
int state;
int identity;
DecodeValueBytes(valueBytes, out headBytes, out entityValBytes, out state, out identity);
var entityItem = new RedisEntityItem()
{
HashId = entityParentKey,
UserId = identity,
KeyBytes = entityKeyBytes,
ValueBytes = entityValBytes,
State = state,
HasMutilKey = hasMutilKey
};
if (entityItem.State == 1)
{
entityRemList.Add(entityItem);
if (hasMutilKey)
{
mutilKeyMapRemList.Add(entityItem);
}
}
else
{
entityList.Add(entityItem);
if (hasMutilKey)
{
mutilKeyMapList.Add(entityItem);
}
}
if (_enableWriteToDb && isStoreInDb)
{
//增加到Sql等待队列
string sqlWaitQueueKey = GetSqlWaitSyncQueueKey(identity);
sqlWaitSyncQueue.Add(new KeyValuePair <string, byte[][]>(sqlWaitQueueKey, new[] { keyBytes, headBytes }));
}
}
catch
{
redisSyncErrorQueue.Add(new[] { keyBytes, valueBytes });
}
}
var redisErrorKeys = redisSyncErrorQueue.Select(p => p[0]).ToArray();
var redisErrorValues = redisSyncErrorQueue.Select(p => p[1]).ToArray();
var sqlWaitGroups = sqlWaitSyncQueue.GroupBy(p => p.Key);
var setGroups = entityList.GroupBy(p => p.HashId);
var removeGroups = entityRemList.GroupBy(p => p.HashId);
var mutilKeyMapGroups = mutilKeyMapList.GroupBy(p => p.HashId);
var mutilKeyMapRemGroups = mutilKeyMapRemList.GroupBy(p => p.HashId);
RedisConnectionPool.ProcessPipeline(pipeline =>
{
bool hasPost = false;
foreach (var g in setGroups)
{
var entityKeys = g.Select(p => p.KeyBytes).ToArray();
var entityValues = g.Select(p => p.ValueBytes).ToArray();
pipeline.QueueCommand(client => ((RedisClient)client).HMSet(g.Key, entityKeys, entityValues));
hasPost = true;
}
foreach (var g in removeGroups)
{
var keybytes = g.Select(p => p.KeyBytes).ToArray();
pipeline.QueueCommand(client => ((RedisClient)client).HDel(g.Key, keybytes));
hasPost = true;
}
foreach (var g in mutilKeyMapGroups)
{
string hashId = g.Key;
var subGroup = g.GroupBy(t => t.UserId);
foreach (var @group in subGroup)
{
string firstKey = AbstractEntity.EncodeKeyCode(@group.Key.ToString());
var keybytes = @group.Select(p => p.KeyBytes).ToArray();
pipeline.QueueCommand(client => RedisConnectionPool.SetMutilKeyMap((RedisClient)client, hashId, firstKey, keybytes));
hasPost = true;
}
}
foreach (var g in mutilKeyMapRemGroups)
{
string hashId = g.Key;
var subGroup = g.GroupBy(t => t.UserId);
foreach (var @group in subGroup)
{
string firstKey = AbstractEntity.EncodeKeyCode(@group.Key.ToString());
var keybytes = @group.Select(p => p.KeyBytes).ToArray();
pipeline.QueueCommand(client => RedisConnectionPool.RemoveMutilKeyMap((RedisClient)client, hashId, firstKey, keybytes));
hasPost = true;
}
}
if (redisErrorKeys.Length > 0)
{
pipeline.QueueCommand(client => ((RedisClient)client).HMSet(RedisSyncErrorQueueKey, redisErrorKeys, redisErrorValues));
hasPost = true;
}
foreach (var g in sqlWaitGroups)
{
var sqlWaitKeys = g.Select(p => p.Value[0]).ToArray();
var sqlWaitValues = g.Select(p => p.Value[1]).ToArray();
pipeline.QueueCommand(client => ((RedisClient)client).HMSet(g.Key, sqlWaitKeys, sqlWaitValues));
hasPost = true;
}
if (hasPost)
{
pipeline.Flush();
}
result = redisErrorKeys.Length == 0;
});
}
catch (Exception ex)
{
TraceLog.WriteError("DoProcessRedisSyncQueue error:{0}", ex);
try
{
RedisConnectionPool.Process(client => client.HMSet(RedisSyncErrorQueueKey, keys, values));
}
catch (Exception er)
{
TraceLog.WriteError("Put RedisSyncErrorQueue error:{0}", er);
}
}
return(result);
}