/// <summary>
/// 全表同步
/// </summary>
/// <param name="node"></param>
private void ExcuteScanProcess(object obj)
{
var node = obj as SyncNode;
var mongoClient = new Mongo.MongoClient(node.MongoUrl);
var esClient = new EsClient(node.ID, node.EsUrl);
LogUtil.LogInfo(logger, $"全量同步({node.Name})节点开始", node.ID);
try
{
if (!esClient.IsIndexExsit(node.Index))
{
LogUtil.LogInfo(logger, $"检测索引{node.Index}未创建，正在创建...", node.ID);
if (esClient.CreateIndex(node.Index))
{
LogUtil.LogInfo(logger, $"索引{node.Index}创建成功.", node.ID);
}
else
{
throw new Exception($"索引{node.Index}创建失败.");
}
}
LogUtil.LogInfo(logger, $"正在更新类型{node.Type}的mapping", node.ID);
if (esClient.PutMapping(node.Index, node.Type, node.Mapping))
{
LogUtil.LogInfo(logger, $"类型{node.Type}的mapping更新成功", node.ID);
}
else
{
throw new Exception($"类型{node.Type}的mapping更新失败");
}
// 记下当前Oplog的位置
var currentOplog = mongoClient.GetCollectionData <BsonDocument>("local", "oplog.rs", "{}", "{$natural:-1}", 1).FirstOrDefault();
node.OperTailSign = currentOplog["ts"].AsBsonTimestamp.Timestamp;
node.OperTailSignExt = currentOplog["ts"].AsBsonTimestamp.Increment;
client.UpdateCollectionData <SyncNode>(database, collection, node.ID,
Update.Set("OperTailSign", node.OperTailSign).Set("OperTailSignExt", node.OperTailSignExt).ToBsonDocument());
var filter = "{}";
var data = mongoClient.GetCollectionData <BsonDocument>(node.DataBase, node.Collection, filter, limit: 1);
if (node.IsLog && !String.IsNullOrWhiteSpace(node.OperScanSign))
{
filter = data.Last()["_id"].IsObjectId ?
$"{{'_id':{{ $gt:new ObjectId('{node.OperScanSign}')}}}}"
: $"{{'_id':{{ $gt:{node.OperScanSign}}}}}";
data = mongoClient.GetCollectionData <BsonDocument>(node.DataBase, node.Collection, filter, limit: 1000);
}
while (data.Count() > 0)
{
//System.Diagnostics.Stopwatch sw = new System.Diagnostics.Stopwatch();
//sw.Start();
if (esClient.InsertBatchDocument(node.Index, node.Type, IBatchDocuemntHandle(data, node.ProjectFields, node.LinkField)))
{
LogUtil.LogInfo(logger, $"节点({node.Name}),文档(count:{data.Count()})写入ES成功", node.ID);
//if (string.IsNullOrWhiteSpace(node.OperScanSign))
//{
// if (esClient.SetIndexRefreshAndReplia(node.Index, "-1", 0))
// {
// LogUtil.LogInfo(logger, $"ES 索引写入性能优化成功", node.ID);
// }
// else
// {
// LogUtil.LogInfo(logger, $"ES 索引写入性能优化失败", node.ID);
// }
//}
node.Status = SyncStatus.ProcessScan;
node.OperScanSign = data.Last()["_id"].ToString();
//node.OperTailSign = client.GetTimestampFromDateTime(DateTime.UtcNow);
client.UpdateCollectionData <SyncNode>(database, collection, node.ID,
Update.Set("Status", node.Status).Set("OperScanSign", node.OperScanSign).ToBsonDocument());
filter = data.Last()["_id"].IsObjectId ?
$"{{'_id':{{ $gt:new ObjectId('{node.OperScanSign}')}}}}"
: $"{{'_id':{{ $gt:{node.OperScanSign}}}}}";
data = mongoClient.GetCollectionData <BsonDocument>(node.DataBase, node.Collection, filter, limit: 1000);
}
else
{
LogUtil.LogInfo(logger, $"节点({node.Name}),文档(count:{data.Count()})写入ES失败,需手动重置", node.ID);
node.Status = SyncStatus.ScanException;
node.Switch = SyncSwitch.Stop;
client.UpdateCollectionData <SyncNode>(database, collection, node.ID,
Update.Set("Status", node.Status).Set("Switch", node.Switch).ToBsonDocument());
return;
}
if (scanNodesDic.TryGetValue(node.ID, out SyncNode oldNode))
{
node = oldNode;
if (node.Switch == SyncSwitch.Stoping)
{
node.Switch = SyncSwitch.Stop;
client.UpdateCollectionData <SyncNode>(database, collection, node.ID,
Update.Set("Switch", node.Switch).ToBsonDocument());
LogUtil.LogInfo(logger, $"全量同步节点({node.Name})已停止, scan线程停止", node.ID);
return;
}
}
//sw.Stop();
//LogUtil.LogInfo(logger, sw.ElapsedMilliseconds.ToString(), node.ID);
}
if (esClient.SetIndexRefreshAndReplia(node.Index))
{
LogUtil.LogInfo(logger, $"ES 索引{node.Index}副本及刷新时间还原成功", node.ID);
}
else
{
LogUtil.LogInfo(logger, $"ES 索引{node.Index}副本及刷新时间还原失败，可手动还原", node.ID);
}
//if (node.IsLog)
//{
// // 记下当前Oplog的位置
// currentOplog = mongoClient.GetCollectionData<BsonDocument>("local", "oplog.rs", "{}", "{$natural:-1}", 1).FirstOrDefault();
// node.OperTailSign = currentOplog["ts"].AsBsonTimestamp.Timestamp;
// node.OperTailSignExt = currentOplog["ts"].AsBsonTimestamp.Increment;
// client.UpdateCollectionData<SyncNode>(database, collection, node.ID,
// Update.Set("OperTailSign", node.OperTailSign).Set("OperTailSignExt", node.OperTailSignExt).ToBsonDocument());
//}
node.Status = SyncStatus.WaitForTail;
client.UpdateCollectionData <SyncNode>(database, collection, node.ID,
Update.Set("Status", node.Status).ToBsonDocument());
LogUtil.LogInfo(logger, $"索引{node.Index}正在等待增量同步...", node.ID);
}
catch (Exception ex)
{
node.Status = SyncStatus.ScanException;
node.Switch = SyncSwitch.Stop;
client.UpdateCollectionData <SyncNode>(database, collection, node.ID,
Update.Set("Status", node.Status).Set("Switch", node.Switch).ToBsonDocument());
LogUtil.LogError(logger, ex.ToString(), node.ID);
}
LogUtil.LogInfo(logger, $"全量同步({node.Name})节点结束", node.ID);
}