/// <summary>
/// 发送更新数据
/// </summary>
/// <typeparam name="T"></typeparam>
/// <param name="dataList"></param>
/// <param name="sendParam"></param>
public bool SendData <T>(T[] dataList, TransSendParam sendParam) where T : AbstractEntity, new()
{
using (var sender = new SqlDataSender(sendParam.IsChange, sendParam.Schema.ConnectKey))
{
return(sender.Send(dataList));
}
}
/// <summary>
///
/// </summary>
/// <typeparam name="T"></typeparam>
/// <param name="dataList"></param>
/// <param name="sendParam"></param>
public void SendData <T>(T[] dataList, TransSendParam sendParam) where T : AbstractEntity, new()
{
using (var sender = new SqlDataSender(sendParam.IsChange, sendParam.ConnectKey))
{
sender.Send(dataList);
}
}
/// <summary>
/// Send entity to db saved.
/// </summary>
/// <param name="postColumnFunc"></param>
/// <param name="getPropertyFunc"></param>
/// <param name="entityList"></param>
public static void SendToDb <T>(EntityPropertyGetFunc <T> getPropertyFunc, EnttiyPostColumnFunc <T> postColumnFunc, params T[] entityList) where T : ISqlEntity
{
string key = "";
try
{
if (entityList == null || entityList.Length == 0)
{
return;
}
var sender = new SqlDataSender(false);
var groupList = entityList.GroupBy(t => t.GetMessageQueueId());
var sqlList = new List <KeyValuePair <string, KeyValuePair <byte[], long> > >();
foreach (var g in groupList)
{
key = g.Key.ToString();
var valueList = g.ToList();
foreach (var entity in valueList)
{
if (entity == null)
{
continue;
}
SqlStatement statement = sender.GenerateSqlQueue <T>(entity, getPropertyFunc, postColumnFunc);
if (statement == null)
{
throw new Exception(string.Format("Generate sql of \"{0}\" entity error", entity.GetType().FullName));
}
var sqlValueBytes = ProtoBufUtils.Serialize(statement);
string sqlQueueKey = SqlStatementManager.GetSqlQueueKey(statement.IdentityID);
sqlList.Add(new KeyValuePair <string, KeyValuePair <byte[], long> >(sqlQueueKey,
new KeyValuePair <byte[], long>(sqlValueBytes, DateTime.Now.Ticks)));
}
}
RedisConnectionPool.ProcessPipeline(p =>
{
bool hasPost = false;
var groupSqlList = sqlList.GroupBy(t => t.Key);
foreach (var g in groupSqlList)
{
var pairs = g.Select(t => t.Value).ToList();
p.QueueCommand(client => ((RedisClient)client).ZAdd(g.Key, pairs));
hasPost = true;
}
if (hasPost)
{
p.Flush();
}
});
}
catch (Exception ex)
{
TraceLog.WriteError("Send To Db key:{0} error:{1}", key, ex);
}
}
/// <summary>
/// Send entity to db saved.
/// </summary>
/// <param name="entityList"></param>
public static void SendToDb(params AbstractEntity[] entityList)
{
string key = "";
try
{
if (entityList == null || entityList.Length == 0)
{
return;
}
var sender = new SqlDataSender();
var groupList = entityList.GroupBy(t => t.GetIdentityId());
var sqlList = new List <KeyValuePair <string, KeyValuePair <byte[], long> > >();
foreach (var g in groupList)
{
var valueList = g.ToList();
foreach (var entity in valueList)
{
if (entity == null)
{
continue;
}
SqlStatement statement = sender.GenerateSqlQueue(entity);
if (statement == null)
{
throw new Exception(string.Format("Generate sql of \"{0}\" entity error", entity.GetType().FullName));
}
var sqlValueBytes = ProtoBufUtils.Serialize(statement);
string sqlQueueKey = SqlStatementManager.GetSqlQueueKey(statement.IdentityID);
sqlList.Add(new KeyValuePair <string, KeyValuePair <byte[], long> >(sqlQueueKey,
new KeyValuePair <byte[], long>(sqlValueBytes, DateTime.Now.Ticks)));
}
}
RedisConnectionPool.Process(client =>
{
var groupSqlList = sqlList.GroupBy(t => t.Key);
foreach (var g in groupSqlList)
{
var pairs = g.Select(t => t.Value).ToList();
client.ZAdd(g.Key, pairs);
}
});
}
catch (Exception ex)
{
TraceLog.WriteError("Send To Db key:{0} error:{1}", key, ex);
}
}
/// <summary>
/// Process synchronized wait queue of Sql
/// </summary>
/// <param name="waitQueueKey"></param>
/// <param name="keys"></param>
/// <param name="values"></param>
private static void DoProcessSqlWaitSyncQueue(string waitQueueKey, byte[][] keys, byte[][] values)
{
try
{
var sqlSender = new SqlDataSender(false);
IEnumerable <KeyValuePair <string, KeyValuePair <byte[], long> > > sqlList = null;
RedisConnectionPool.ProcessPipeline(client =>
{
sqlList = GenerateSqlFrom(sqlSender, client, keys, values);
}, p =>
{
if (sqlList == null)
{
return(0);
}
var keyValuePairs = sqlList.ToList();
var groupSqlList = keyValuePairs.GroupBy(t => t.Key);
int sqlCount = keyValuePairs.Count();
bool hasPost = false;
long result = 0;
foreach (var g in groupSqlList)
{
var pairs = g.Select(t => t.Value).ToList();
p.QueueCommand(c => ((RedisClient)c).ZAdd(g.Key, pairs), r =>
{//onsuccess, 已经存在的返回值为0
result += r;
ProfileManager.PostSqlOfMessageQueueTimes(g.Key, sqlCount);
});
hasPost = true;
}
if (hasPost)
{
p.Flush();
}
return(result);
});
}
catch (Exception ex)
{
TraceLog.WriteError("DoProcessSqlWaitSyncQueue error:{0}", ex);
}
}
/// <summary>
/// Process synchronized wait queue of Sql
/// </summary>
/// <param name="waitQueueKey"></param>
/// <param name="keys"></param>
/// <param name="values"></param>
private static void DoProcessSqlWaitSyncQueue(string waitQueueKey, byte[][] keys, byte[][] values)
{
try
{
var sqlSender = new SqlDataSender();
RedisConnectionPool.Process(client =>
{
var sqlList = GenerateSqlFrom(sqlSender, client, keys, values);
var groupSqlList = sqlList.GroupBy(t => t.Key);
foreach (var g in groupSqlList)
{
var pairs = g.Select(t => t.Value).ToList();
client.ZAdd(g.Key, pairs);
}
});
}
catch (Exception ex)
{
TraceLog.WriteError("DoProcessSqlWaitSyncQueue error:{0}", ex);
}
}
/// <summary>
/// Generate Sql statements from the Keys-Values
/// </summary>
/// <param name="sender"></param>
/// <param name="client"></param>
/// <param name="keys"></param>
/// <param name="values"></param>
/// <returns></returns>
private static IEnumerable <KeyValuePair <string, KeyValuePair <byte[], long> > > GenerateSqlFrom(SqlDataSender sender, RedisClient client, byte[][] keys, byte[][] values)
{
var typeKeyValuePairs = new List <KeyValuePair <string, byte[][]> >();
for (int i = 0; i < keys.Length; i++)
{
byte[] keyBytes = keys[i];
byte[] headBytes = values[i];
string entityTypeKey = RedisConnectionPool.ToStringKey(keyBytes);
string[] entityKeys = entityTypeKey.Split(',')[0].Split('_');
string typeName = entityKeys[0];
byte[] entityKeyBytes = RedisConnectionPool.ToByteKey(entityKeys[1]);
typeKeyValuePairs.Add(new KeyValuePair <string, byte[][]>(typeName, new[] { entityKeyBytes, keyBytes, headBytes })
);
}
var sqlList = new List <KeyValuePair <string, KeyValuePair <byte[], long> > >();
var typeGroup = typeKeyValuePairs.GroupBy(p => p.Key);
foreach (var g in typeGroup)
{
var typeName = g.Key;
try
{
string entityParentKey = RedisConnectionPool.GetRedisEntityKeyName(typeName);
string asmName = "";
var enitityAsm = EntitySchemaSet.EntityAssembly;
if (enitityAsm != null)
{
asmName = "," + enitityAsm.GetName().Name;
}
Type type = Type.GetType(string.Format("{0}{1}", RedisConnectionPool.DecodeTypeName(typeName), asmName), false, true);
if (type == null)
{
//调试模式下type为空处理
type = enitityAsm != null?enitityAsm.GetType(RedisConnectionPool.DecodeTypeName(typeName), false, true) : null;
if (type == null)
{
throw new ArgumentTypeException(string.Format("Get entity \"{0}\" type is null", entityParentKey));
}
}
var keyBuffers = g.Select(p => p.Value[0]).ToArray();
var headBuffers = g.Select(p => p.Value[2]).ToArray();
var valueBuffers = client.HMGet(entityParentKey, keyBuffers);
for (int i = 0; i < keyBuffers.Length; i++)
{
string keyCode = RedisConnectionPool.ToStringKey(keyBuffers[i]);
var buffer = valueBuffers != null && valueBuffers.Length > i ? valueBuffers[i] : null;
var headBytes = headBuffers[i];
int identity;
int state;
DecodeHeadBytes(headBytes, out identity, out state);
AbstractEntity entity = null;
if (state == 1 && buffer == null)
{
//entity remove ops
entity = type.CreateInstance(new object[0]) as AbstractEntity;
entity.IsDelete = true;
entity.SetKeyValue(keyCode, typeName);
}
else if (buffer != null)
{
entity = _serializer.Deserialize(buffer, type) as AbstractEntity;
}
if (entity != null)
{
if (state == 1)
{
entity.IsDelete = true;
}
SqlStatement statement = sender.GenerateSqlQueue(entity);
if (statement == null)
{
throw new Exception(string.Format("Generate sql of \"{0}\" entity error", typeName));
}
var sqlValueBytes = ProtoBufUtils.Serialize(statement);
string sqlQueueKey = SqlStatementManager.GetSqlQueueKey(statement.IdentityID);
sqlList.Add(new KeyValuePair <string, KeyValuePair <byte[], long> >(sqlQueueKey,
new KeyValuePair <byte[], long>(sqlValueBytes, DateTime.Now.Ticks)));
}
else
{
throw new Exception(string.Format("Get \"{0}\" entity is null, keycode:{1},buffer len:{2},state:{3}",
typeName, keyCode, buffer == null ? -1 : buffer.Length, state));
}
}
}
catch (Exception er)
{
TraceLog.WriteError("FindEntityFromRedis {0} error:{1}", typeName, er);
var errorKeys = g.Select(p => p.Value[1]).ToArray();
var errorValues = g.Select(p => p.Value[2]).ToArray();
client.HMSet(SqlSyncWaitErrirQueueKey, errorKeys, errorValues);
}
}
return(sqlList);
}
private static List<KeyValuePair<string, KeyValuePair<byte[], long>>> FindEntityFromRedis(SqlDataSender sender, RedisClient client, byte[][] keys, byte[][] values)
{
var typeKeyValuePairs = new List<KeyValuePair<string, byte[][]>>();
for (int i = 0; i < keys.Length; i++)
{
byte[] keyBytes = keys[i];
byte[] headBytes = values[i];
string entityTypeKey = RedisConnectionPool.ToStringKey(keyBytes);
string[] entityKeys = entityTypeKey.Split(',')[0].Split('_');
string typeName = entityKeys[0];
byte[] entityKeyBytes = RedisConnectionPool.ToByteKey(entityKeys[1]);
typeKeyValuePairs.Add(
new KeyValuePair<string, byte[][]>(typeName, new[] { entityKeyBytes, keyBytes, headBytes })
);
}
var sqlList = new List<KeyValuePair<string, KeyValuePair<byte[], long>>>();
var typeGroup = typeKeyValuePairs.GroupBy(p => p.Key);
foreach (var g in typeGroup)
{
var typeName = g.Key;
try
{
string entityParentKey = RedisConnectionPool.GetRedisEntityKeyName(typeName);
string asmName = "";
var enitityAsm = EntitySchemaSet.EntityAssembly;
if (enitityAsm != null)
{
asmName = "," + enitityAsm.GetName().Name;
}
Type type = Type.GetType(string.Format("{0}{1}", typeName, asmName), false, true);
if (type == null)
{
//调试模式下type为空处理
type = enitityAsm.GetType(typeName, false, true);
if (type == null)
{
throw new ArgumentTypeException(string.Format("Get entity \"{0}\" type is null", entityParentKey));
}
}
var keyBuffers = g.Select(p => p.Value[0]).ToArray();
var headBuffers = g.Select(p => p.Value[2]).ToArray();
var valueBuffers = client.HMGet(entityParentKey, keyBuffers);
for (int i = 0; i < keyBuffers.Length; i++)
{
string keyCode = RedisConnectionPool.ToStringKey(keyBuffers[i]);
var buffer = valueBuffers[i];
var headBytes = headBuffers[i];
int identity;
int state;
DecodeHeadBytes(headBytes, out identity, out state);
AbstractEntity entity = null;
if (state == 1 && buffer == null)
{
//entity remove ops
entity = type.CreateInstance(new object[0]) as AbstractEntity;
entity.IsDelete = true;
entity.SetKeyValue(keyCode, typeName);
}
else if (buffer != null)
{
entity = ProtoBufUtils.Deserialize(buffer, type) as AbstractEntity;
}
if (entity != null)
{
if (state == 1) entity.IsDelete = true;
SqlStatement statement = sender.GenerateSqlQueue(entity);
var sqlValueBytes = ProtoBufUtils.Serialize(statement);
string sqlQueueKey = SqlStatementManager.GetSqlQueueKey(statement.IdentityID);
sqlList.Add(new KeyValuePair<string, KeyValuePair<byte[], long>>(sqlQueueKey,
new KeyValuePair<byte[], long>(sqlValueBytes, DateTime.Now.Ticks)));
}
else
{
throw new Exception(string.Format("Get \"{0}\" entity is null, keycode:{1},buffer len:{2}",
typeName, keyCode, buffer == null ? -1 : buffer.Length));
}
}
}
catch (Exception er)
{
TraceLog.WriteError("FindEntityFromRedis {0} error:{1}", typeName, er);
var errorKeys = g.Select(p => p.Value[1]).ToArray();
var errorValues = g.Select(p => p.Value[2]).ToArray();
client.HMSet(SqlSyncWaitErrirQueueKey, errorKeys, errorValues);
}
}
return sqlList;
}
private static void DoProcessSqlWaitSyncQueue(string waitQueueKey, byte[][] keys, byte[][] values)
{
try
{
var sqlSender = new SqlDataSender();
RedisConnectionPool.Process(client =>
{
var sqlList = FindEntityFromRedis(sqlSender, client, keys, values);
var groupSqlList = sqlList.GroupBy(t => t.Key);
foreach (var g in groupSqlList)
{
var pairs = g.Select(t => t.Value).ToList();
client.ZAdd(g.Key, pairs);
}
});
}
catch (Exception ex)
{
TraceLog.WriteError("DoProcessSqlWaitSyncQueue error:{0}", ex);
}
}
/// <summary>
/// Send entity to db saved.
/// </summary>
/// <param name="postColumnFunc"></param>
/// <param name="getPropertyFunc"></param>
/// <param name="entityList"></param>
public static void SendToDb <T>(string connectKey, EntityPropertyGetFunc <T> getPropertyFunc, EnttiyPostColumnFunc <T> postColumnFunc, params T[] entityList) where T : ISqlEntity
{
string key = "";
try
{
if (entityList == null || entityList.Length == 0)
{
return;
}
var sender = new SqlDataSender(false, connectKey);
var groupList = entityList.GroupBy(t => t.GetMessageQueueId());
var sqlList = new List <KeyValuePair <string, byte[]> >();
foreach (var g in groupList)
{
key = g.Key.ToString();
var valueList = g.ToList();
foreach (var entity in valueList)
{
if (entity == null)
{
continue;
}
SqlStatement statement = sender.GenerateSqlQueue <T>(entity, getPropertyFunc, postColumnFunc);
if (statement == null)
{
throw new Exception(string.Format("Generate sql of \"{0}\" entity error", entity.GetType().FullName));
}
var sqlValueBytes = ProtoBufUtils.Serialize(statement);
string sqlQueueKey = SqlStatementManager.GetSqlQueueKey(statement.IdentityID);
//修复顺序重复导致Sql执行乱序问题
sqlList.Add(new KeyValuePair <string, byte[]>(sqlQueueKey, sqlValueBytes));
TraceLog.WriteDebug($"Send to polls[{sqlQueueKey}]-> sql:{statement.ToString()}");
}
}
RedisConnectionPool.ProcessPipeline(p =>
{
bool hasPost = false;
var groupSqlList = sqlList.GroupBy(t => t.Key);
int sqlCount = sqlList.Count;
foreach (var g in groupSqlList)
{
var values = g.Select(t => t.Value).ToArray();
p.QueueCommand(client => ((RedisClient)client).LPush(g.Key, values), () =>
{//onSuccess
ProfileManager.PostSqlOfMessageQueueTimes(g.Key, sqlCount);
});
hasPost = true;
}
if (hasPost)
{
p.Flush();
}
});
}
catch (Exception ex)
{
TraceLog.WriteError("Send To Db key:{0} error:{1}", key, ex);
}
}
/// <summary>
/// Send entity to db saved.
/// </summary>
/// <param name="entityList"></param>
public static void SendToDb(params AbstractEntity[] entityList)
{
string key = "";
try
{
if (entityList == null || entityList.Length == 0) return;
var sender = new SqlDataSender();
var groupList = entityList.GroupBy(t => t.GetIdentityId());
var sqlList = new List<KeyValuePair<string, KeyValuePair<byte[], long>>>();
foreach (var g in groupList)
{
var valueList = g.ToList();
foreach (var entity in valueList)
{
if (entity == null) continue;
SqlStatement statement = sender.GenerateSqlQueue(entity);
if (statement == null)
{
throw new Exception(string.Format("Generate sql of \"{0}\" entity error", entity.GetType().FullName));
}
var sqlValueBytes = ProtoBufUtils.Serialize(statement);
string sqlQueueKey = SqlStatementManager.GetSqlQueueKey(statement.IdentityID);
sqlList.Add(new KeyValuePair<string, KeyValuePair<byte[], long>>(sqlQueueKey,
new KeyValuePair<byte[], long>(sqlValueBytes, DateTime.Now.Ticks)));
}
}
RedisConnectionPool.Process(client =>
{
var groupSqlList = sqlList.GroupBy(t => t.Key);
foreach (var g in groupSqlList)
{
var pairs = g.Select(t => t.Value).ToList();
client.ZAdd(g.Key, pairs);
}
});
}
catch (Exception ex)
{
TraceLog.WriteError("Send To Db key:{0} error:{1}", key, ex);
}
}
