/// <summary>
/// 增量同步
/// </summary>
/// <param name="obj"></param>
private void ExcuteTailProcess(object obj)
{
var node = obj as SyncNode;
var mongoClient = new Mongo.MongoClient(node.MongoUrl);
var esClient = new EsClient(node.ID, node.EsUrl);
LogUtil.LogInfo(logger, $"增量同步({node.Name})节点开始", node.ID);
// 动态规划
int maxCount = 1000;
int minElapsedMilliseconds = 150;
int maxElapsedMilliseconds = 3500;
bool bulkSwitch = false;
DateTime lastDataTime = DateTime.Now;
List <EsData> esDatas = new List <EsData>();
try
{
node.Status = SyncStatus.ProcessTail;
client.UpdateCollectionData <SyncNode>(database, collection, node.ID,
Update.Set("Status", node.Status).ToBsonDocument());
while (true)
{
using (var cursor = mongoClient.TailMongoOpLogs($"{node.DataBase}.{node.Collection}", node.OperTailSign, node.OperTailSignExt))
{
try
{
#region version 1
/*
* foreach (var opLog in cursor.ToEnumerable())
* {
* if (!opArr.Contains(opLog["op"].AsString)) continue;
*
* if (tailNodesDic.TryGetValue(node.ID, out SyncNode oldNode))
* {
* node = oldNode;
* if (node.Switch == SyncSwitch.Stoping)
* {
* node.Switch = SyncSwitch.Stop;
* client.UpdateCollectionData<SyncNode>(database, collection, node.ID,
* Update.Set("Switch", node.Switch).ToBsonDocument());
* LogUtil.LogInfo(logger, $"增量同步节点({node.Name})已停止, tail线程停止", node.ID);
* return;
* }
* }
*
* bool flag = true;
* switch (opLog["op"].AsString)
* {
* case "i":
* var iid = string.IsNullOrWhiteSpace(node.LinkField) ? opLog["o"]["_id"].ToString() : opLog["o"][node.LinkField].ToString();
* var idoc = IDocuemntHandle(opLog["o"].AsBsonDocument, node.ProjectFields);
* if (idoc.Names.Count() > 0)
* {
* if (string.IsNullOrWhiteSpace(node.LinkField))
* {
* if (esClient.InsertDocument(node.Index, node.Type, iid, idoc))
* LogUtil.LogInfo(logger, $"节点({node.Name}),文档（id:{iid}）写入ES成功", node.ID);
* else
* {
* flag = false;
* LogUtil.LogInfo(logger, $"节点({node.Name}),文档（id:{iid}）写入ES失败", node.ID);
* }
* }
* else
* {
* idoc.Remove("id");
* if (esClient.UpdateDocument(node.Index, node.Type, iid, idoc))
* LogUtil.LogInfo(logger, $"节点({node.Name}),文档（id:{iid}）更新ES成功", node.ID);
* else
* {
* flag = false;
* LogUtil.LogInfo(logger, $"节点({node.Name}),文档（id:{iid}）更新ES失败", node.ID);
* }
* }
* }
* break;
* case "u":
* var uid = opLog["o2"]["_id"].ToString();
* var udoc = opLog["o"].AsBsonDocument;
*
* if (!string.IsNullOrWhiteSpace(node.LinkField))
* {
* var filter = opLog["o2"]["_id"].IsObjectId ? $"{{'_id':new ObjectId('{uid}')}}" : $"{{'_id':{uid}}}";
* var dataDetail = mongoClient.GetCollectionData<BsonDocument>(node.DataBase, node.Collection, filter, limit: 1).FirstOrDefault();
* if (dataDetail == null || !dataDetail.Contains(node.LinkField)) continue;
* uid = dataDetail[node.LinkField].ToString();
* }
*
* if (udoc.Contains("$unset"))
* {
* var unsetdoc = udoc["$unset"].AsBsonDocument;
* udoc.Remove("$unset");
*
* var delFields = UnsetDocHandle(unsetdoc, node.ProjectFields);
* if (delFields.Count > 0)
* {
* if (esClient.DeleteField(node.Index, node.Type, uid, delFields))
* {
* LogUtil.LogInfo(logger, $"节点({node.Name}),文档（id:{uid}）删除ES字段({string.Join(",", delFields)})成功", node.ID);
* }
* else
* {
* flag = false;
* LogUtil.LogInfo(logger, $"节点({node.Name}),文档（id:{uid}）删除ES字段({string.Join(",", delFields)})失败", node.ID);
* break;
* }
* }
* }
*
* udoc = UDocuemntHandle(udoc, node.ProjectFields);
* if (udoc.Names.Count() > 0)
* {
* if (esClient.UpdateDocument(node.Index, node.Type, uid, udoc))
* LogUtil.LogInfo(logger, $"节点({node.Name}),文档（id:{uid}）更新ES成功", node.ID);
* else
* {
* flag = false;
* LogUtil.LogInfo(logger, $"节点({node.Name}),文档（id:{uid}）更新ES失败", node.ID);
* }
* }
*
* break;
* case "d":
* var did = opLog["o"]["_id"].ToString();
* if (string.IsNullOrWhiteSpace(node.LinkField))
* {
* if (esClient.DeleteDocument(node.Index, node.Type, did))
* LogUtil.LogInfo(logger, $"节点({node.Name}),文档（id:{did}）删除ES成功", node.ID);
* else
* {
* flag = false;
* LogUtil.LogInfo(logger, $"节点({node.Name}),文档（id:{did}）删除ES失败", node.ID);
* }
* }
* break;
* default:
* break;
* }
*
* if (flag)
* {
* node.OperTailSign = opLog["ts"].AsBsonTimestamp.Timestamp;
* node.OperTailSignExt = opLog["ts"].AsBsonTimestamp.Increment;
* client.UpdateCollectionData<SyncNode>(database, collection, node.ID,
* Update.Set("OperTailSign", node.OperTailSign).Set("OperTailSignExt", node.OperTailSignExt).ToBsonDocument());
* }
* else
* {
* node.Status = SyncStatus.TailException;
* node.Switch = SyncSwitch.Stop;
* client.UpdateCollectionData<SyncNode>(database, collection, node.ID,
* Update.Set("Status", node.Status).Set("Switch", node.Switch).ToBsonDocument());
*
* return;
* }
* }*/
#endregion
#region version now
foreach (var opLog in cursor.ToEnumerable())
{
//LogUtil.LogInfo(logger, "开始", node.ID);
//System.Diagnostics.Stopwatch sw = new System.Diagnostics.Stopwatch();
//sw.Start();
if (tailNodesDic.TryGetValue(node.ID, out SyncNode oldNode))
{
node = oldNode;
if (node.Switch == SyncSwitch.Stoping)
{
node.Switch = SyncSwitch.Stop;
client.UpdateCollectionData <SyncNode>(database, collection, node.ID,
Update.Set("Switch", node.Switch).ToBsonDocument());
LogUtil.LogInfo(logger, $"增量同步节点({node.Name})已停止, tail线程停止", node.ID);
return;
}
}
if (!opLog["ns"].AsString.Equals($"{node.DataBase}.{node.Collection}"))
{
continue;
}
if (!opArr.Contains(opLog["op"].AsString))
{
continue;
}
switch (opLog["op"].AsString)
{
case "i":
var iid = string.IsNullOrWhiteSpace(node.LinkField) ? opLog["o"]["_id"].ToString() : opLog["o"][node.LinkField].ToString();
var idoc = IDocuemntHandle(opLog["o"].AsBsonDocument, node.ProjectFields);
if (idoc.Names.Count() > 0)
{
if (!string.IsNullOrWhiteSpace(node.LinkField))
{
idoc.Remove("id");
}
esDatas.Add(new EsData()
{
Oper = "insert",
ID = iid,
Data = idoc,
Time = DateTime.Now
});
}
break;
case "u":
var uid = opLog["o2"]["_id"].ToString();
var udoc = opLog["o"].AsBsonDocument;
if (!string.IsNullOrWhiteSpace(node.LinkField))
{
var filter = opLog["o2"]["_id"].IsObjectId ? $"{{'_id':new ObjectId('{uid}')}}" : $"{{'_id':{uid}}}";
var dataDetail = mongoClient.GetCollectionData <BsonDocument>(node.DataBase, node.Collection, filter, limit: 1).FirstOrDefault();
if (dataDetail == null || !dataDetail.Contains(node.LinkField))
{
continue;
}
uid = dataDetail[node.LinkField].ToString();
}
if (udoc.Contains("$unset"))
{
var unsetdoc = udoc["$unset"].AsBsonDocument;
udoc.Remove("$unset");
var delFields = UnsetDocHandle(unsetdoc, node.ProjectFields);
if (delFields.Count > 0)
{
esDatas.Add(new EsData()
{
Oper = "delFields",
ID = uid,
Data = delFields,
Time = DateTime.Now
});
}
}
udoc = UDocuemntHandle(udoc, node.ProjectFields);
if (udoc.Names.Count() > 0)
{
esDatas.Add(new EsData()
{
Oper = "update",
ID = uid,
Data = udoc,
Time = DateTime.Now
});
}
break;
case "d":
var did = opLog["o"]["_id"].ToString();
if (string.IsNullOrWhiteSpace(node.LinkField))
{
esDatas.Add(new EsData()
{
Oper = "delete",
ID = did,
Time = DateTime.Now
});
}
break;
default:
break;
}
if (esDatas.Count > 0)
{
if (bulkSwitch)
{
//LogUtil.LogInfo(logger, (DateTime.Now - esDatas.First().Time).TotalMilliseconds.ToString(), node.ID);
if (esDatas.Count >= maxCount)
{
}
else if ((DateTime.Now - esDatas.First().Time).TotalMilliseconds >= maxElapsedMilliseconds)
{
bulkSwitch = false;
//LogUtil.LogInfo(logger, $"节点({node.Name})增量同步降级为单条同步", node.ID);
}
else
{
continue;
}
}
else
{
if ((DateTime.Now - lastDataTime).TotalMilliseconds <= minElapsedMilliseconds)
{
bulkSwitch = true;
//LogUtil.LogInfo(logger, $"节点({node.Name})增量同步升级为批量同步", node.ID);
}
}
lastDataTime = DateTime.Now;
// bulk
if (esClient.InsertBatchDocument(node.Index, node.Type, BatchDocuemntHandle(esDatas)))
{
LogUtil.LogInfo(logger, $"节点({node.Name}),文档(count:{esDatas.Count})更新ES成功", node.ID);
esDatas.Clear();
node.OperTailSign = opLog["ts"].AsBsonTimestamp.Timestamp;
node.OperTailSignExt = opLog["ts"].AsBsonTimestamp.Increment;
client.UpdateCollectionData <SyncNode>(database, collection, node.ID,
Update.Set("OperTailSign", node.OperTailSign).Set("OperTailSignExt", node.OperTailSignExt).ToBsonDocument());
}
else
{
LogUtil.LogInfo(logger, $"节点({node.Name}),文档(count:{esDatas.Count})更新ES失败,需手动重置", node.ID);
node.Status = SyncStatus.TailException;
node.Switch = SyncSwitch.Stop;
client.UpdateCollectionData <SyncNode>(database, collection, node.ID,
Update.Set("Status", node.Status).Set("Switch", node.Switch).ToBsonDocument());
return;
}
}
//sw.Stop();
//LogUtil.LogInfo(logger, sw.ElapsedMilliseconds.ToString(), node.ID);
//LogUtil.LogInfo(logger, "结束", node.ID);
}
#endregion
}
catch (MongoExecutionTimeoutException ex)
{
// Nohandle with MongoExecutionTimeoutException
if (node != null)
{
LogUtil.LogWarn(logger, $"同步({node.Name})节点异常：{ex}", node.ID);
mongoClient = new Mongo.MongoClient(node.MongoUrl);
}
}
catch (MongoCommandException ex)
{
// Nohandle with MongoExecutionTimeoutException
if (node != null)
{
LogUtil.LogWarn(logger, $"同步({node.Name})节点异常：{ex}", node.ID);
mongoClient = new Mongo.MongoClient(node.MongoUrl);
}
}
}
}
}
catch (Exception ex)
{
node.Status = SyncStatus.TailException;
node.Switch = SyncSwitch.Stop;
client.UpdateCollectionData <SyncNode>(database, collection, node.ID,
Update.Set("Status", node.Status).Set("Switch", node.Switch).ToBsonDocument());
LogUtil.LogError(logger, $"同步({node.Name})节点异常：{ex}", node.ID);
}
LogUtil.LogInfo(logger, $"增量同步({node.Name})节点结束", node.ID);
}
public SyncClient(string mongoUrl)
{
this.client = new Mongo.MongoClient(mongoUrl);
}
/// <summary>
/// 全表同步
/// </summary>
/// <param name="node"></param>
private void ExcuteScanProcess(object obj)
{
var node = obj as SyncNode;
var mongoClient = new Mongo.MongoClient(node.MongoUrl);
var esClient = new EsClient(node.ID, node.EsUrl);
LogUtil.LogInfo(logger, $"全量同步({node.Name})节点开始", node.ID);
try
{
if (!esClient.IsIndexExsit(node.Index))
{
LogUtil.LogInfo(logger, $"检测索引{node.Index}未创建，正在创建...", node.ID);
if (esClient.CreateIndex(node.Index))
{
LogUtil.LogInfo(logger, $"索引{node.Index}创建成功.", node.ID);
}
else
{
throw new Exception($"索引{node.Index}创建失败.");
}
}
LogUtil.LogInfo(logger, $"正在更新类型{node.Type}的mapping", node.ID);
if (esClient.PutMapping(node.Index, node.Type, node.Mapping))
{
LogUtil.LogInfo(logger, $"类型{node.Type}的mapping更新成功", node.ID);
}
else
{
throw new Exception($"类型{node.Type}的mapping更新失败");
}
// 记下当前Oplog的位置
var currentOplog = mongoClient.GetCollectionData <BsonDocument>("local", "oplog.rs", "{}", "{$natural:-1}", 1).FirstOrDefault();
node.OperTailSign = currentOplog["ts"].AsBsonTimestamp.Timestamp;
node.OperTailSignExt = currentOplog["ts"].AsBsonTimestamp.Increment;
client.UpdateCollectionData <SyncNode>(database, collection, node.ID,
Update.Set("OperTailSign", node.OperTailSign).Set("OperTailSignExt", node.OperTailSignExt).ToBsonDocument());
var filter = "{}";
var data = mongoClient.GetCollectionData <BsonDocument>(node.DataBase, node.Collection, filter, limit: 1);
if (node.IsLog && !String.IsNullOrWhiteSpace(node.OperScanSign))
{
filter = data.Last()["_id"].IsObjectId ?
$"{{'_id':{{ $gt:new ObjectId('{node.OperScanSign}')}}}}"
: $"{{'_id':{{ $gt:{node.OperScanSign}}}}}";
data = mongoClient.GetCollectionData <BsonDocument>(node.DataBase, node.Collection, filter, limit: 1000);
}
while (data.Count() > 0)
{
//System.Diagnostics.Stopwatch sw = new System.Diagnostics.Stopwatch();
//sw.Start();
if (esClient.InsertBatchDocument(node.Index, node.Type, IBatchDocuemntHandle(data, node.ProjectFields, node.LinkField)))
{
LogUtil.LogInfo(logger, $"节点({node.Name}),文档(count:{data.Count()})写入ES成功", node.ID);
//if (string.IsNullOrWhiteSpace(node.OperScanSign))
//{
// if (esClient.SetIndexRefreshAndReplia(node.Index, "-1", 0))
// {
// LogUtil.LogInfo(logger, $"ES 索引写入性能优化成功", node.ID);
// }
// else
// {
// LogUtil.LogInfo(logger, $"ES 索引写入性能优化失败", node.ID);
// }
//}
node.Status = SyncStatus.ProcessScan;
node.OperScanSign = data.Last()["_id"].ToString();
//node.OperTailSign = client.GetTimestampFromDateTime(DateTime.UtcNow);
client.UpdateCollectionData <SyncNode>(database, collection, node.ID,
Update.Set("Status", node.Status).Set("OperScanSign", node.OperScanSign).ToBsonDocument());
filter = data.Last()["_id"].IsObjectId ?
$"{{'_id':{{ $gt:new ObjectId('{node.OperScanSign}')}}}}"
: $"{{'_id':{{ $gt:{node.OperScanSign}}}}}";
data = mongoClient.GetCollectionData <BsonDocument>(node.DataBase, node.Collection, filter, limit: 1000);
}
else
{
LogUtil.LogInfo(logger, $"节点({node.Name}),文档(count:{data.Count()})写入ES失败,需手动重置", node.ID);
node.Status = SyncStatus.ScanException;
node.Switch = SyncSwitch.Stop;
client.UpdateCollectionData <SyncNode>(database, collection, node.ID,
Update.Set("Status", node.Status).Set("Switch", node.Switch).ToBsonDocument());
return;
}
if (scanNodesDic.TryGetValue(node.ID, out SyncNode oldNode))
{
node = oldNode;
if (node.Switch == SyncSwitch.Stoping)
{
node.Switch = SyncSwitch.Stop;
client.UpdateCollectionData <SyncNode>(database, collection, node.ID,
Update.Set("Switch", node.Switch).ToBsonDocument());
LogUtil.LogInfo(logger, $"全量同步节点({node.Name})已停止, scan线程停止", node.ID);
return;
}
}
//sw.Stop();
//LogUtil.LogInfo(logger, sw.ElapsedMilliseconds.ToString(), node.ID);
}
if (esClient.SetIndexRefreshAndReplia(node.Index))
{
LogUtil.LogInfo(logger, $"ES 索引{node.Index}副本及刷新时间还原成功", node.ID);
}
else
{
LogUtil.LogInfo(logger, $"ES 索引{node.Index}副本及刷新时间还原失败，可手动还原", node.ID);
}
//if (node.IsLog)
//{
// // 记下当前Oplog的位置
// currentOplog = mongoClient.GetCollectionData<BsonDocument>("local", "oplog.rs", "{}", "{$natural:-1}", 1).FirstOrDefault();
// node.OperTailSign = currentOplog["ts"].AsBsonTimestamp.Timestamp;
// node.OperTailSignExt = currentOplog["ts"].AsBsonTimestamp.Increment;
// client.UpdateCollectionData<SyncNode>(database, collection, node.ID,
// Update.Set("OperTailSign", node.OperTailSign).Set("OperTailSignExt", node.OperTailSignExt).ToBsonDocument());
//}
node.Status = SyncStatus.WaitForTail;
client.UpdateCollectionData <SyncNode>(database, collection, node.ID,
Update.Set("Status", node.Status).ToBsonDocument());
LogUtil.LogInfo(logger, $"索引{node.Index}正在等待增量同步...", node.ID);
}
catch (Exception ex)
{
node.Status = SyncStatus.ScanException;
node.Switch = SyncSwitch.Stop;
client.UpdateCollectionData <SyncNode>(database, collection, node.ID,
Update.Set("Status", node.Status).Set("Switch", node.Switch).ToBsonDocument());
LogUtil.LogError(logger, ex.ToString(), node.ID);
}
LogUtil.LogInfo(logger, $"全量同步({node.Name})节点结束", node.ID);
}