public SyncResponse Exec() { var startedOn = DateTime.UtcNow; log.Info("process started at " + startedOn.NormalizedFormat()); client = new ElasticsearchClient(_config.ElasticSearchConfiguration); using (var _lock = new SyncLock(client, LogIndex, LockType)) { DateTime? lastSyncDate = ConfigureIncrementalProcess(_config.SqlCommand, _config.ColumnsToCompareWithLastSyncDate); var data = GetSerializedObject(); log.Info(String.Format("{0} objects have been serialized.", data.Count())); var syncResponse = new SyncResponse(startedOn); syncResponse = IndexProcess(data, syncResponse); if (_config.DeleteConfiguration != null) { _config.SqlConnection.Open(); Dictionary<object, Dictionary<string, object>> deleteData = null; if (lastSyncDate != null) ConfigureIncrementalProcess(_config.DeleteConfiguration.SqlCommand, _config.DeleteConfiguration.ColumnsToCompareWithLastSyncDate, lastSyncDate); using (SqlDataReader rdr = _config.DeleteConfiguration.SqlCommand.ExecuteReader()) { deleteData = rdr.Serialize(); } _config.SqlConnection.Close(); syncResponse = DeleteProcess(deleteData, syncResponse); } syncResponse = Log(syncResponse); log.Info(String.Format("process duration: {0}ms", Math.Truncate((syncResponse.EndedOn - syncResponse.StartedOn).TotalMilliseconds))); return syncResponse; } }
public SyncResponse Exec(bool force = false) { try { var startedOn = DateTime.UtcNow; log.Debug("process started at " + startedOn.NormalizedFormat()); client = new ElasticsearchClient(_config.ElasticSearchConfiguration); using (var _lock = new SyncLock(client, LogIndex, LockType, force)) { DateTime?lastSyncDate = ConfigureIncrementalProcess(_config.SqlCommand, _config.ColumnsToCompareWithLastSyncDate); log.Info(String.Format("last sync date: {0}", lastSyncDate != null ? lastSyncDate.ToString() : "null")); var syncResponse = new SyncResponse(startedOn); //DELETE PROCESS if (_config.DeleteConfiguration != null) { _config.SqlConnection.Open(); Dictionary <object, Dictionary <string, object> > deleteData = null; if (lastSyncDate != null) { ConfigureIncrementalProcess(_config.DeleteConfiguration.SqlCommand, _config.DeleteConfiguration.ColumnsToCompareWithLastSyncDate, lastSyncDate); } using (SqlDataReader rdr = _config.DeleteConfiguration.SqlCommand.ExecuteReader()) { deleteData = rdr.Serialize(); } _config.SqlConnection.Close(); syncResponse = DeleteProcess(deleteData, syncResponse); } //INDEX PROCESS if (_config.SqlCommand != null) { var dataCount = 0; try { _config.SqlConnection.Open(); if (_config.PageSize.HasValue) { var page = 0; var size = _config.PageSize; var commandText = _config.SqlCommand.CommandText; while (true) { var conditionBuilder = new StringBuilder("("); conditionBuilder .Append("RowNumber BETWEEN ") .Append(page * size + 1) .Append(" AND ") .Append(page * size + size) .Append(")"); _config.SqlCommand.CommandText = AddSqlCondition(commandText, conditionBuilder.ToString()); var pageData = GetSerializedObject(); var pageDataCount = pageData.Count(); dataCount += pageDataCount; log.Info(String.Format("{0} objects have been serialized from page {1}.", pageDataCount, page)); IndexProcess(pageData, syncResponse); pageData.Clear(); pageData = null; GC.Collect(GC.MaxGeneration); if (pageDataCount < size) { break; } page++; } } else { var data = GetSerializedObject(); dataCount = data.Count(); IndexProcess(data, syncResponse); } log.Info(String.Format("{0} objects have been serialized.", dataCount)); } finally { _config.SqlConnection.Close(); } } //LOG PROCESS syncResponse = LogProcess(syncResponse); log.Debug(String.Format("process duration: {0}ms", Math.Truncate((syncResponse.EndedOn - syncResponse.StartedOn).TotalMilliseconds))); return(syncResponse); } } catch (Exception ex) { log.Error("an error has occurred: " + ex); throw ex; } }