/// <summary> /// 增量同步 /// </summary> /// <param name="obj"></param> private void ExcuteTailProcess(object obj) { var node = obj as SyncNode; var mongoClient = new Mongo.MongoClient(node.MongoUrl); var esClient = new EsClient(node.ID, node.EsUrl); LogUtil.LogInfo(logger, $"增量同步({node.Name})节点开始", node.ID); // 动态规划 int maxCount = 1000; int minElapsedMilliseconds = 150; int maxElapsedMilliseconds = 3500; bool bulkSwitch = false; DateTime lastDataTime = DateTime.Now; List <EsData> esDatas = new List <EsData>(); try { node.Status = SyncStatus.ProcessTail; client.UpdateCollectionData <SyncNode>(database, collection, node.ID, Update.Set("Status", node.Status).ToBsonDocument()); while (true) { using (var cursor = mongoClient.TailMongoOpLogs($"{node.DataBase}.{node.Collection}", node.OperTailSign, node.OperTailSignExt)) { try { #region version 1 /* * foreach (var opLog in cursor.ToEnumerable()) * { * if (!opArr.Contains(opLog["op"].AsString)) continue; * * if (tailNodesDic.TryGetValue(node.ID, out SyncNode oldNode)) * { * node = oldNode; * if (node.Switch == SyncSwitch.Stoping) * { * node.Switch = SyncSwitch.Stop; * client.UpdateCollectionData<SyncNode>(database, collection, node.ID, * Update.Set("Switch", node.Switch).ToBsonDocument()); * LogUtil.LogInfo(logger, $"增量同步节点({node.Name})已停止, tail线程停止", node.ID); * return; * } * } * * bool flag = true; * switch (opLog["op"].AsString) * { * case "i": * var iid = string.IsNullOrWhiteSpace(node.LinkField) ? opLog["o"]["_id"].ToString() : opLog["o"][node.LinkField].ToString(); * var idoc = IDocuemntHandle(opLog["o"].AsBsonDocument, node.ProjectFields); * if (idoc.Names.Count() > 0) * { * if (string.IsNullOrWhiteSpace(node.LinkField)) * { * if (esClient.InsertDocument(node.Index, node.Type, iid, idoc)) * LogUtil.LogInfo(logger, $"节点({node.Name}),文档(id:{iid})写入ES成功", node.ID); * else * { * flag = false; * LogUtil.LogInfo(logger, $"节点({node.Name}),文档(id:{iid})写入ES失败", node.ID); * } * } * else * { * idoc.Remove("id"); * if (esClient.UpdateDocument(node.Index, node.Type, iid, idoc)) * LogUtil.LogInfo(logger, $"节点({node.Name}),文档(id:{iid})更新ES成功", node.ID); * else * { * flag = false; * LogUtil.LogInfo(logger, $"节点({node.Name}),文档(id:{iid})更新ES失败", node.ID); * } * } * } * break; * case "u": * var uid = opLog["o2"]["_id"].ToString(); * var udoc = opLog["o"].AsBsonDocument; * * if (!string.IsNullOrWhiteSpace(node.LinkField)) * { * var filter = opLog["o2"]["_id"].IsObjectId ? $"{{'_id':new ObjectId('{uid}')}}" : $"{{'_id':{uid}}}"; * var dataDetail = mongoClient.GetCollectionData<BsonDocument>(node.DataBase, node.Collection, filter, limit: 1).FirstOrDefault(); * if (dataDetail == null || !dataDetail.Contains(node.LinkField)) continue; * uid = dataDetail[node.LinkField].ToString(); * } * * if (udoc.Contains("$unset")) * { * var unsetdoc = udoc["$unset"].AsBsonDocument; * udoc.Remove("$unset"); * * var delFields = UnsetDocHandle(unsetdoc, node.ProjectFields); * if (delFields.Count > 0) * { * if (esClient.DeleteField(node.Index, node.Type, uid, delFields)) * { * LogUtil.LogInfo(logger, $"节点({node.Name}),文档(id:{uid})删除ES字段({string.Join(",", delFields)})成功", node.ID); * } * else * { * flag = false; * LogUtil.LogInfo(logger, $"节点({node.Name}),文档(id:{uid})删除ES字段({string.Join(",", delFields)})失败", node.ID); * break; * } * } * } * * udoc = UDocuemntHandle(udoc, node.ProjectFields); * if (udoc.Names.Count() > 0) * { * if (esClient.UpdateDocument(node.Index, node.Type, uid, udoc)) * LogUtil.LogInfo(logger, $"节点({node.Name}),文档(id:{uid})更新ES成功", node.ID); * else * { * flag = false; * LogUtil.LogInfo(logger, $"节点({node.Name}),文档(id:{uid})更新ES失败", node.ID); * } * } * * break; * case "d": * var did = opLog["o"]["_id"].ToString(); * if (string.IsNullOrWhiteSpace(node.LinkField)) * { * if (esClient.DeleteDocument(node.Index, node.Type, did)) * LogUtil.LogInfo(logger, $"节点({node.Name}),文档(id:{did})删除ES成功", node.ID); * else * { * flag = false; * LogUtil.LogInfo(logger, $"节点({node.Name}),文档(id:{did})删除ES失败", node.ID); * } * } * break; * default: * break; * } * * if (flag) * { * node.OperTailSign = opLog["ts"].AsBsonTimestamp.Timestamp; * node.OperTailSignExt = opLog["ts"].AsBsonTimestamp.Increment; * client.UpdateCollectionData<SyncNode>(database, collection, node.ID, * Update.Set("OperTailSign", node.OperTailSign).Set("OperTailSignExt", node.OperTailSignExt).ToBsonDocument()); * } * else * { * node.Status = SyncStatus.TailException; * node.Switch = SyncSwitch.Stop; * client.UpdateCollectionData<SyncNode>(database, collection, node.ID, * Update.Set("Status", node.Status).Set("Switch", node.Switch).ToBsonDocument()); * * return; * } * }*/ #endregion #region version now foreach (var opLog in cursor.ToEnumerable()) { //LogUtil.LogInfo(logger, "开始", node.ID); //System.Diagnostics.Stopwatch sw = new System.Diagnostics.Stopwatch(); //sw.Start(); if (tailNodesDic.TryGetValue(node.ID, out SyncNode oldNode)) { node = oldNode; if (node.Switch == SyncSwitch.Stoping) { node.Switch = SyncSwitch.Stop; client.UpdateCollectionData <SyncNode>(database, collection, node.ID, Update.Set("Switch", node.Switch).ToBsonDocument()); LogUtil.LogInfo(logger, $"增量同步节点({node.Name})已停止, tail线程停止", node.ID); return; } } if (!opLog["ns"].AsString.Equals($"{node.DataBase}.{node.Collection}")) { continue; } if (!opArr.Contains(opLog["op"].AsString)) { continue; } switch (opLog["op"].AsString) { case "i": var iid = string.IsNullOrWhiteSpace(node.LinkField) ? opLog["o"]["_id"].ToString() : opLog["o"][node.LinkField].ToString(); var idoc = IDocuemntHandle(opLog["o"].AsBsonDocument, node.ProjectFields); if (idoc.Names.Count() > 0) { if (!string.IsNullOrWhiteSpace(node.LinkField)) { idoc.Remove("id"); } esDatas.Add(new EsData() { Oper = "insert", ID = iid, Data = idoc, Time = DateTime.Now }); } break; case "u": var uid = opLog["o2"]["_id"].ToString(); var udoc = opLog["o"].AsBsonDocument; if (!string.IsNullOrWhiteSpace(node.LinkField)) { var filter = opLog["o2"]["_id"].IsObjectId ? $"{{'_id':new ObjectId('{uid}')}}" : $"{{'_id':{uid}}}"; var dataDetail = mongoClient.GetCollectionData <BsonDocument>(node.DataBase, node.Collection, filter, limit: 1).FirstOrDefault(); if (dataDetail == null || !dataDetail.Contains(node.LinkField)) { continue; } uid = dataDetail[node.LinkField].ToString(); } if (udoc.Contains("$unset")) { var unsetdoc = udoc["$unset"].AsBsonDocument; udoc.Remove("$unset"); var delFields = UnsetDocHandle(unsetdoc, node.ProjectFields); if (delFields.Count > 0) { esDatas.Add(new EsData() { Oper = "delFields", ID = uid, Data = delFields, Time = DateTime.Now }); } } udoc = UDocuemntHandle(udoc, node.ProjectFields); if (udoc.Names.Count() > 0) { esDatas.Add(new EsData() { Oper = "update", ID = uid, Data = udoc, Time = DateTime.Now }); } break; case "d": var did = opLog["o"]["_id"].ToString(); if (string.IsNullOrWhiteSpace(node.LinkField)) { esDatas.Add(new EsData() { Oper = "delete", ID = did, Time = DateTime.Now }); } break; default: break; } if (esDatas.Count > 0) { if (bulkSwitch) { //LogUtil.LogInfo(logger, (DateTime.Now - esDatas.First().Time).TotalMilliseconds.ToString(), node.ID); if (esDatas.Count >= maxCount) { } else if ((DateTime.Now - esDatas.First().Time).TotalMilliseconds >= maxElapsedMilliseconds) { bulkSwitch = false; //LogUtil.LogInfo(logger, $"节点({node.Name})增量同步降级为单条同步", node.ID); } else { continue; } } else { if ((DateTime.Now - lastDataTime).TotalMilliseconds <= minElapsedMilliseconds) { bulkSwitch = true; //LogUtil.LogInfo(logger, $"节点({node.Name})增量同步升级为批量同步", node.ID); } } lastDataTime = DateTime.Now; // bulk if (esClient.InsertBatchDocument(node.Index, node.Type, BatchDocuemntHandle(esDatas))) { LogUtil.LogInfo(logger, $"节点({node.Name}),文档(count:{esDatas.Count})更新ES成功", node.ID); esDatas.Clear(); node.OperTailSign = opLog["ts"].AsBsonTimestamp.Timestamp; node.OperTailSignExt = opLog["ts"].AsBsonTimestamp.Increment; client.UpdateCollectionData <SyncNode>(database, collection, node.ID, Update.Set("OperTailSign", node.OperTailSign).Set("OperTailSignExt", node.OperTailSignExt).ToBsonDocument()); } else { LogUtil.LogInfo(logger, $"节点({node.Name}),文档(count:{esDatas.Count})更新ES失败,需手动重置", node.ID); node.Status = SyncStatus.TailException; node.Switch = SyncSwitch.Stop; client.UpdateCollectionData <SyncNode>(database, collection, node.ID, Update.Set("Status", node.Status).Set("Switch", node.Switch).ToBsonDocument()); return; } } //sw.Stop(); //LogUtil.LogInfo(logger, sw.ElapsedMilliseconds.ToString(), node.ID); //LogUtil.LogInfo(logger, "结束", node.ID); } #endregion } catch (MongoExecutionTimeoutException ex) { // Nohandle with MongoExecutionTimeoutException if (node != null) { LogUtil.LogWarn(logger, $"同步({node.Name})节点异常:{ex}", node.ID); mongoClient = new Mongo.MongoClient(node.MongoUrl); } } catch (MongoCommandException ex) { // Nohandle with MongoExecutionTimeoutException if (node != null) { LogUtil.LogWarn(logger, $"同步({node.Name})节点异常:{ex}", node.ID); mongoClient = new Mongo.MongoClient(node.MongoUrl); } } } } } catch (Exception ex) { node.Status = SyncStatus.TailException; node.Switch = SyncSwitch.Stop; client.UpdateCollectionData <SyncNode>(database, collection, node.ID, Update.Set("Status", node.Status).Set("Switch", node.Switch).ToBsonDocument()); LogUtil.LogError(logger, $"同步({node.Name})节点异常:{ex}", node.ID); } LogUtil.LogInfo(logger, $"增量同步({node.Name})节点结束", node.ID); }
/// <summary> /// 全表同步 /// </summary> /// <param name="node"></param> private void ExcuteScanProcess(object obj) { var node = obj as SyncNode; var mongoClient = new Mongo.MongoClient(node.MongoUrl); var esClient = new EsClient(node.ID, node.EsUrl); LogUtil.LogInfo(logger, $"全量同步({node.Name})节点开始", node.ID); try { if (!esClient.IsIndexExsit(node.Index)) { LogUtil.LogInfo(logger, $"检测索引{node.Index}未创建,正在创建...", node.ID); if (esClient.CreateIndex(node.Index)) { LogUtil.LogInfo(logger, $"索引{node.Index}创建成功.", node.ID); } else { throw new Exception($"索引{node.Index}创建失败."); } } LogUtil.LogInfo(logger, $"正在更新类型{node.Type}的mapping", node.ID); if (esClient.PutMapping(node.Index, node.Type, node.Mapping)) { LogUtil.LogInfo(logger, $"类型{node.Type}的mapping更新成功", node.ID); } else { throw new Exception($"类型{node.Type}的mapping更新失败"); } // 记下当前Oplog的位置 var currentOplog = mongoClient.GetCollectionData <BsonDocument>("local", "oplog.rs", "{}", "{$natural:-1}", 1).FirstOrDefault(); node.OperTailSign = currentOplog["ts"].AsBsonTimestamp.Timestamp; node.OperTailSignExt = currentOplog["ts"].AsBsonTimestamp.Increment; client.UpdateCollectionData <SyncNode>(database, collection, node.ID, Update.Set("OperTailSign", node.OperTailSign).Set("OperTailSignExt", node.OperTailSignExt).ToBsonDocument()); var filter = "{}"; var data = mongoClient.GetCollectionData <BsonDocument>(node.DataBase, node.Collection, filter, limit: 1); if (node.IsLog && !String.IsNullOrWhiteSpace(node.OperScanSign)) { filter = data.Last()["_id"].IsObjectId ? $"{{'_id':{{ $gt:new ObjectId('{node.OperScanSign}')}}}}" : $"{{'_id':{{ $gt:{node.OperScanSign}}}}}"; data = mongoClient.GetCollectionData <BsonDocument>(node.DataBase, node.Collection, filter, limit: 1000); } while (data.Count() > 0) { //System.Diagnostics.Stopwatch sw = new System.Diagnostics.Stopwatch(); //sw.Start(); if (esClient.InsertBatchDocument(node.Index, node.Type, IBatchDocuemntHandle(data, node.ProjectFields, node.LinkField))) { LogUtil.LogInfo(logger, $"节点({node.Name}),文档(count:{data.Count()})写入ES成功", node.ID); //if (string.IsNullOrWhiteSpace(node.OperScanSign)) //{ // if (esClient.SetIndexRefreshAndReplia(node.Index, "-1", 0)) // { // LogUtil.LogInfo(logger, $"ES 索引写入性能优化成功", node.ID); // } // else // { // LogUtil.LogInfo(logger, $"ES 索引写入性能优化失败", node.ID); // } //} node.Status = SyncStatus.ProcessScan; node.OperScanSign = data.Last()["_id"].ToString(); //node.OperTailSign = client.GetTimestampFromDateTime(DateTime.UtcNow); client.UpdateCollectionData <SyncNode>(database, collection, node.ID, Update.Set("Status", node.Status).Set("OperScanSign", node.OperScanSign).ToBsonDocument()); filter = data.Last()["_id"].IsObjectId ? $"{{'_id':{{ $gt:new ObjectId('{node.OperScanSign}')}}}}" : $"{{'_id':{{ $gt:{node.OperScanSign}}}}}"; data = mongoClient.GetCollectionData <BsonDocument>(node.DataBase, node.Collection, filter, limit: 1000); } else { LogUtil.LogInfo(logger, $"节点({node.Name}),文档(count:{data.Count()})写入ES失败,需手动重置", node.ID); node.Status = SyncStatus.ScanException; node.Switch = SyncSwitch.Stop; client.UpdateCollectionData <SyncNode>(database, collection, node.ID, Update.Set("Status", node.Status).Set("Switch", node.Switch).ToBsonDocument()); return; } if (scanNodesDic.TryGetValue(node.ID, out SyncNode oldNode)) { node = oldNode; if (node.Switch == SyncSwitch.Stoping) { node.Switch = SyncSwitch.Stop; client.UpdateCollectionData <SyncNode>(database, collection, node.ID, Update.Set("Switch", node.Switch).ToBsonDocument()); LogUtil.LogInfo(logger, $"全量同步节点({node.Name})已停止, scan线程停止", node.ID); return; } } //sw.Stop(); //LogUtil.LogInfo(logger, sw.ElapsedMilliseconds.ToString(), node.ID); } if (esClient.SetIndexRefreshAndReplia(node.Index)) { LogUtil.LogInfo(logger, $"ES 索引{node.Index}副本及刷新时间还原成功", node.ID); } else { LogUtil.LogInfo(logger, $"ES 索引{node.Index}副本及刷新时间还原失败,可手动还原", node.ID); } //if (node.IsLog) //{ // // 记下当前Oplog的位置 // currentOplog = mongoClient.GetCollectionData<BsonDocument>("local", "oplog.rs", "{}", "{$natural:-1}", 1).FirstOrDefault(); // node.OperTailSign = currentOplog["ts"].AsBsonTimestamp.Timestamp; // node.OperTailSignExt = currentOplog["ts"].AsBsonTimestamp.Increment; // client.UpdateCollectionData<SyncNode>(database, collection, node.ID, // Update.Set("OperTailSign", node.OperTailSign).Set("OperTailSignExt", node.OperTailSignExt).ToBsonDocument()); //} node.Status = SyncStatus.WaitForTail; client.UpdateCollectionData <SyncNode>(database, collection, node.ID, Update.Set("Status", node.Status).ToBsonDocument()); LogUtil.LogInfo(logger, $"索引{node.Index}正在等待增量同步...", node.ID); } catch (Exception ex) { node.Status = SyncStatus.ScanException; node.Switch = SyncSwitch.Stop; client.UpdateCollectionData <SyncNode>(database, collection, node.ID, Update.Set("Status", node.Status).Set("Switch", node.Switch).ToBsonDocument()); LogUtil.LogError(logger, ex.ToString(), node.ID); } LogUtil.LogInfo(logger, $"全量同步({node.Name})节点结束", node.ID); }