private async Task SaveSyncData(IDbConnection connection, ISyncableItemInfo remoteSyncableItemInfo, SyncStatus status) { var data = JObject.Parse("{item:{itemRefs:[]}}"); if (!remoteSyncableItemInfo.Deleted) { var request = new JObject { { "sessionID", _remoteSessionId }, { "item", SyncUtil.SyncableItemInfoToJson(remoteSyncableItemInfo) } }; var response = await _transport.TransportAsync(SyncEndpoint.GetItemData, request); data = response; } SessionDbHelper.SaveItemData(connection, remoteSyncableItemInfo, status, data); var itemRefs = (JArray)data["item"]["itemRefs"]; foreach (var item in itemRefs) { ISyncableItemInfo itemRefInfo = SyncUtil.SyncableItemInfoFromJson(item); ISyncableItemInfo localItemRefInfo = _store.LocateCurrentItemInfo(itemRefInfo); if (localItemRefInfo != null && localItemRefInfo.Deleted) { await SaveSyncData(connection, itemRefInfo, SyncStatus.MayBeNeeded); } } }
public void ResolveConflictLocalWins(SyncConflict conflict) { if (_closed) { throw new InvalidOperationException(); } var tickCount = _store.IncrementLocalRepilcaTickCount(); var modifiedReplica = new ReplicaInfo { ReplicaId = _store.GetLocalReplicaId(), ReplicaTickCount = tickCount }; var resolvedStatus = conflict.LocalItemInfo.Deleted ? SyncStatus.Delete : SyncStatus.Update; var data = JObject.Parse("{item:{itemRefs:[]}}"); if (resolvedStatus != SyncStatus.Delete) { var builder = SyncUtil.JsonItemFromSyncableItemInfo(conflict.LocalItemInfo); _store.BuildItemData(conflict.LocalItemInfo, builder); data = new JObject { { "item", builder } }; } using (var connection = _syncSessionDbConnectionProvider.GetSyncSessionDbConnection(_localSessionId)) { SessionDbHelper.ResolveItemWithData(connection, conflict.RemoteItemInfo, resolvedStatus, modifiedReplica, data); } }
private void CommitChanges() { if (_remoteKnowledge == null) { return; } ReportProgressAndCheckCacellation(new SyncProgress { Stage = SyncStage.ApplyingChangesLocally, PercentComplete = 0, Message = "Applying changes locally" }); using ( IDbConnection connection = _syncSessionDbConnectionProvider.GetSyncSessionDbConnection(_localSessionId)) { SyncUtil.ApplyChangesAndUpdateKnowledge(connection, _store, _remoteKnowledge); } ReportProgressAndCheckCacellation(new SyncProgress { Stage = SyncStage.ApplyingChangesLocally, PercentComplete = 100, Message = "Applied all changes locally" }); }
public async Task LoadVocs_NewSession_SendFromClient_DoctorOk() { // sync voc var voc = sSession.Get <Vocabulary>(IntToGuid <Vocabulary>(2)); await s.OnlySelectedVocs(sSession, voc.ToEnumerable()) .SendFrom(Side.Server, Scope.Voc); // close app SyncUtil.Clear(); // sync user bool errorWas = false; EventHandler <StringEventArgs> h = (s1, e) => { errorWas = e.str.Contains("Error"); }; Poster.MessagePosted += h; await s.WithoutCustomVocsInDoc().SendFrom(Side.Client, Scope.User); Poster.MessagePosted -= h; // doctor table already provisioned in user scope Assert.IsTrue(!errorWas); }
internal void Scale(double factor, TimeSpan baseTime) { PreciseStart = SyncUtil.Scale(PreciseStart, baseTime, factor); PreciseEnd = SyncUtil.Scale(PreciseEnd, baseTime, factor); subtitle.UpdateFramesFromTimes(); }
public void Sync(SyncOptions options) { var totalIndexedUsers = GetAllUsers(options.UserLimit, options.UserAccountFilter); Console.WriteLine(@"""fullSync"":{""getUserProfileByIndex"":" + totalIndexedUsers.Count + ","); Console.WriteLine(string.Concat(@"""users"" : [", string.Join(",\n", totalIndexedUsers), "]")); SyncUtil.JsonClose(","); }
// // ****** Methods for adding unsorted data // public void AddUnsortedDataSet(DataSet dataSet) { // for each table in the DataSet foreach (DataTable curTable in dataSet.Tables) { int numColumns = curTable.Columns.Count; // this table should already be in the list of added // tables SortedTable sortedTable; if (!_sortedTables.TryGetValue(curTable.TableName, out sortedTable)) { SyncTracer.Error("Cannot Apply Changes since Adapters are missing for the following tables: {0}. " + "Please ensure that the local and global names on the Adapters are set properly.", curTable); throw new DbSyncException("MissingProviderAdapter"); } if (sortedTable._schema == null) { // add a new row storage dictionary and schema if we // need one sortedTable._schema = curTable.Clone(); Debug.Assert(sortedTable._schema.DataSet == null); sortedTable._rows = new SortedDictionary <SyncId, SortedRow>(_syncIdComparer); } if (curTable.Rows.Count == 0) { continue; } object[] idColVals = new object[sortedTable._idCols.Count]; // for each row foreach (DataRow curRow in curTable.Rows) { DataRowVersion viewVersion = (curRow.RowState == DataRowState.Deleted) ? DataRowVersion.Original : DataRowVersion.Current; for (int idx = 0; idx < idColVals.Length; idx += 1) { idColVals[idx] = curRow[sortedTable._idCols[idx], viewVersion]; } // Add the row to this tables row storage dictionary SyncId curRowId = SyncUtil.InitRowId(curTable.TableName, idColVals); // Note: There is an issue with batching in the provider which causes the same primary key to be repeated across files. // This crashes the .Add call below. To work-around this problem, we need to check if the key already exists. // If it does, then remove it and add it again so that the latest record is always used. if (sortedTable._rows.ContainsKey(curRowId)) { sortedTable._rows.Remove(curRowId); } sortedTable._rows.Add(curRowId, new SortedRow(curRow, numColumns)); } } }
public void Sync(SyncOptions options) { var totalIndexedUsers = GetChanges(options); Console.WriteLine(@"""incrementalSync"":{""userProfileChangeServiceGetChanges"":" + totalIndexedUsers.Count + ","); Console.WriteLine(@"""changeToken"":""" + _changeToken + @""","); Console.WriteLine(string.Concat(@"""users"" : [", string.Join(",\n", totalIndexedUsers), "]")); SyncUtil.JsonClose(); }
public void Sync(SyncOptions options) { NetworkCredential nc = null; if (!string.IsNullOrEmpty(_domain) && !string.IsNullOrEmpty(_username) && !string.IsNullOrEmpty(_password)) { nc = new NetworkCredential { Domain = _domain, UserName = _username, Password = _password }; SyncUtil.WriteLine("Using Credentials for {0}\\{1}.", _domain, _username); } else { SyncUtil.WriteLine("Using Default Credentials."); } try { if (Settings.Default.ConsoleUPS_MyProfileUPSService_UserProfileService.StartsWith(DefaultSite)) { SyncUtil.WriteLine("Verify ConsoleUPS.exe.config settings before running."); return; } using (var ups = new UserProfileService { PreAuthenticate = false }) { if (nc != null) { ups.Credentials = nc; } else { ups.UseDefaultCredentials = true; } var total = ups.GetUserProfileCount(); Console.WriteLine(@"""getUserProfileCount"":" + total + ","); SyncFull.Instance(ups).Sync(options); SyncIncremental.Instance(ups).Sync(options); } } catch (Exception ex) { if (ex.Message.Contains("status 401")) { SyncUtil.WriteLine("Login failed please check ConsoleUPS.exe.config"); } throw; } }
private List <string> GetChanges(SyncOptions options) { var users = new List <string>(); try { _changeToken = options.IgnoreChangeToken ? string.Empty : Settings.Default.ChangeToken; var changeTokenStart = new UserProfileChangeToken(); var profileChanges = _changeService.GetChanges(_changeToken, new UserProfileChangeQuery { ChangeTokenStart = changeTokenStart, Add = true, Update = true, UserProfile = true, SingleValueProperty = true, MultiValueProperty = true, }); Settings.Default.ChangeToken = profileChanges.ChangeToken; Settings.Default.Save(); var accountNameChanges = profileChanges.Changes.GroupBy(d => d.UserAccountName).Select(gr => gr.Key).ToList(); foreach (var accountChanged in accountNameChanges) { try { var userProfile = _ups.GetUserProfileByName(accountChanged); if (options.UserLimit > 0 && users.Count >= options.UserLimit) { return(users); } if (IsValidProfile(userProfile, options.UserAccountFilter)) { users.Add(FieldsToJson(userProfile)); } } catch (Exception ex) { SyncUtil.WriteLine("Error : {0} AccountChanged: {1}", ex.Message, accountChanged); } } } catch (Exception ex) { SyncUtil.WriteLine("IncrementalSyncGetChangesFailed : {0} {1}", ex.Message, ex.StackTrace); Settings.Default.ChangeToken = string.Empty; Settings.Default.Save(); } return(users); }
private List <string> GetAllUsers(int userLimit, string userAccountFilter = null) { var users = new List <string>(); var nextIndex = -1; try { GetUserProfileByIndexResult userInstance; do { userInstance = _ups.GetUserProfileByIndex(nextIndex); if (userInstance == null || userInstance.UserProfile == null) { continue; } try { if (userLimit > 0 && users.Count >= userLimit) { return(users); } if (IsValidProfile(userInstance.UserProfile, userAccountFilter)) { users.Add(FieldsToJson(userInstance.UserProfile)); if (users.Count % 100 == 0) { SyncUtil.WriteLine("Saving {0] users.", users.Count); } } } catch (Exception ex) { var fieldCount = userInstance.UserProfile.Length; SyncUtil.WriteLine("Error : {0} FieldCount: {1}", ex.Message, fieldCount); } var nextValue = userInstance.NextValue ?? string.Empty; if (!int.TryParse(nextValue.Replace(",", ""), out nextIndex)) { SyncUtil.WriteLine("Error with next index : {0}", nextValue); } }while (userInstance != null && userInstance.UserProfile != null); } catch (Exception ex) { SyncUtil.WriteLine("FullSyncGetUserProfileByIndexFailed : {0} {1}", ex.Message, ex.StackTrace); } return(users); }
public override void BuildItemData(ISyncableItemInfo itemInfo, JObject builder) { IBook book = Adapter.BookRepository.getBookByRowID(GetRowIdFromItemInfo(itemInfo)); builder.Add("title", book.Title); JArray authors = new JArray(); foreach (IPerson author in book.Authors) { ReplicaItemId id = ((Person)author).Id; authors.Add(SyncUtil.GenerateItemRefAndIndex(builder, getSyncableItemInfoFrom(id))); } builder.Add("authors", authors); }
/// <summary> /// Creates a Range set build and pre calculates the effective /// tables ranges for use later when building a BatchRangeSet. /// </summary> /// <param name="idFormat"> /// The id format for the SyncIds /// </param> /// <param name="tableNames"> /// All the table names in that will be in the sum of the /// batches. Order does not matter. /// </param> public BatchRangeSetBuilder(SyncIdFormat idFormat, List <string> tableNames) { _tableRanges = new Dictionary <string, BatchRange>(tableNames.Count); _idFormat = idFormat; int numTables = tableNames.Count; SortedList <SyncId, string> tablesSortedBySyncId = new SortedList <SyncId, string>(numTables); List <object> emptyPkVals = new List <object>(0); foreach (string curTable in tableNames) { SyncId idBeforeTable = SyncUtil.InitRowId(curTable, emptyPkVals); tablesSortedBySyncId.Add(idBeforeTable, curTable); } // for each table we need to determine the: // - starting SyncId (zero or first id in table) // - ending SyncId (just before next table or infinity) BatchRange prevTableRange = null; foreach (KeyValuePair <SyncId, string> curElem in tablesSortedBySyncId) { BatchRange curTableRange = new BatchRange(); curTableRange.TableName = curElem.Value; // assume this is the last table and then fix this if // there is another table curTableRange.End = _idFormat.Infinity; if (prevTableRange == null) { // first table starts at zero curTableRange.Start = _idFormat.Zero; } else { // fix up prev end to be correct // set it's end to be one before current tables // starting SyncId prevTableRange.End = curElem.Key; curTableRange.Start = IdPlusOne(_idFormat, curElem.Key); } prevTableRange = curTableRange; _tableRanges.Add(curTableRange.TableName, curTableRange); } }
public override void SaveItemData(ISyncableItemInfo itemInfo, JObject itemData) { long rowId = GetRowIdFromItemInfo(itemInfo); IDbCommand command = Adapter.Connection.CreateCommand(); if (rowId == -1) { command.CommandText = "INSERT INTO Books(BookTitle, CreatedReplica, CreatedTickCount, ModifiedReplica, ModifiedTickCount ) VALUES(@BookTitle, @CreatedReplica, @CreatedTickCount, @ModifiedReplica, @ModifiedTickCount)"; command.AddParameter("@CreatedReplica", Adapter.GetLocalReplicaIdForGlobalReplicaId(itemInfo.Created.ReplicaId)); command.AddParameter("@CreatedTickCount", itemInfo.Created.ReplicaTickCount); } else { command.CommandText = "UPDATE Books SET BookTitle=@BookTitle,ModifiedReplica=@ModifiedReplica, ModifiedTickCount=@ModifiedTickCount WHERE BookID=@BookID"; command.AddParameter("@BookID", rowId); } command.AddParameter("@BookTitle", (string)itemData["item"]["title"]); command.AddParameter("@ModifiedReplica", Adapter.GetLocalReplicaIdForGlobalReplicaId(itemInfo.Modified.ReplicaId)); command.AddParameter("@ModifiedTickCount", itemInfo.Modified.ReplicaTickCount); command.ExecuteNonQuery(); command.Parameters.Clear(); if (rowId == -1) { rowId = GetRowIdFromItemInfo(itemInfo); } command.CommandText = "DELETE FROM BookAuthors WHERE BookID = @ID;"; command.AddParameter("@ID", rowId); command.ExecuteNonQuery(); command.Parameters.Clear(); int authorPriority = 0; foreach (var authors in itemData["item"]["authors"]) { ISyncableItemInfo authorItemInfo = SyncUtil.SyncableItemInfoFromJsonItemRef(itemData["item"], authors); long authorRowId = Adapter.HandlerForItemType(authorItemInfo.ItemType).GetRowIdFromItemInfo(authorItemInfo); authorPriority++; command.CommandText = String.Format("INSERT INTO BookAuthors(BookID, PersonID, AuthorPriority) VALUES ({0},{1},{2})", rowId, authorRowId, authorPriority); command.ExecuteNonQuery(); } }
private async Task <int> SaveChangesBatch(IDbConnection connection, IList <IReplicaInfo> localKnowledge, int startItem) { var request = new JObject { { "sessionID", _remoteSessionId }, { "startItem", startItem }, { "maxBatchCount", PullMaxBatchCount }, { "maxBatchSize", PullMaxBatchSize } }; JObject response = await _transport.TransportAsync(SyncEndpoint.GetItemDataBatch, request); var batch = (JArray)response["batch"]; foreach (var item in batch) { ISyncableItemInfo remoteSyncableItemInfo = SyncUtil.SyncableItemInfoFromJson(item["item"]); var itemData = new JObject { { "item", item["item"] } }; var status = SyncStatus.MayBeNeeded; if (!SyncUtil.KnowledgeContains(localKnowledge, remoteSyncableItemInfo.Modified)) { ISyncableItemInfo localSyncableItemInfo = _store.LocateCurrentItemInfo(remoteSyncableItemInfo); status = SyncUtil.CalculateSyncStatus(remoteSyncableItemInfo, localSyncableItemInfo, _remoteKnowledge); } SessionDbHelper.SaveItemData(connection, remoteSyncableItemInfo, status, itemData); var itemRefs = (JArray)itemData["item"]["itemRefs"]; foreach (var itemRef in itemRefs) { ISyncableItemInfo itemRefInfo = SyncUtil.SyncableItemInfoFromJson(itemRef); ISyncableItemInfo localItemRefInfo = _store.LocateCurrentItemInfo(itemRefInfo); if (localItemRefInfo != null && localItemRefInfo.Deleted) { await SaveSyncData(connection, itemRefInfo, SyncStatus.MayBeNeeded); } } } return(batch.Count); }
public virtual string FieldsToJson(IEnumerable<PropertyData> properties) { var fields = new List<string>(); var currentName = string.Empty; foreach (var propertyData in properties) { try { if (propertyData == null) continue; currentName = propertyData.Name; fields.Add(string.Format(@"""{0}"" : ""{1}""", propertyData.Name.Replace(@"""", "'"), GetValueData(propertyData.Values.FirstOrDefault()))); } catch (Exception ex) { SyncUtil.WriteLine("{0} Warning : {1}", currentName, ex.Message); } } return string.Concat("{", string.Join(",", fields), "}"); }
public string GetDbState() { JObject state = new JObject(); var knowledge = GenerateLocalKnowledge().OrderBy((ri) => { return(ri.ReplicaId + ri.ReplicaTickCount.ToString()); }); state.Add("knowledge", SyncUtil.KnowledgeToJson(knowledge)); foreach (var itemType in GetItemTypes()) { var handler = HandlerForItemType(itemType); JArray items = new JArray(); IDbCommand selectCommand = _connection.CreateCommand(); selectCommand.CommandText = String.Format("SELECT CreatedReplica, CreatedTickCount, ModifiedReplica, ModifiedTickCount FROM {0}", handler.DbTable); IList <ISyncableItemInfo> itemInfos = new List <ISyncableItemInfo>(); using (IDataReader reader = selectCommand.ExecuteReader()) { while (reader.Read()) { IReplicaInfo createdReplicaInfo = ReplicaInfoFromDataReader(reader, "Created"); IReplicaInfo modifiedReplicaInfo = ReplicaInfoFromDataReader(reader, "Modified"); itemInfos.Add(new SyncableItemInfo { ItemType = handler.TypeName, Created = createdReplicaInfo, Modified = modifiedReplicaInfo, Deleted = false }); } } var sortedItemInfos = itemInfos.OrderBy((ii) => { return(ii.Created.ReplicaId + ii.Created.ReplicaTickCount.ToString()); }); foreach (var syncItemInfo in sortedItemInfos) { JObject builder = SyncUtil.JsonItemFromSyncableItemInfo(syncItemInfo); BuildItemData(syncItemInfo, builder); items.Add(builder); } state.Add(itemType, items); } return(state.ToString()); }
public override DuplicateStatus GetDuplicateStatus(JObject localItemData, JObject remoteItemData) { string localName = (string)localItemData["item"]["title"]; string remoteName = (string)remoteItemData["item"]["title"]; if (localName != remoteName) { return(DuplicateStatus.None); } JArray localAuthors = (JArray)localItemData["item"]["authors"]; JArray remoteAuthors = (JArray)remoteItemData["item"]["authors"]; if (localAuthors.Count != remoteAuthors.Count) { return(DuplicateStatus.None); } for (int i = 0; i < localAuthors.Count; i++) { ISyncableItemInfo localAuthorItemInfo = SyncUtil.SyncableItemInfoFromJsonItemRef(localItemData["item"], localAuthors[i]); ISyncableItemInfo remoteAuthorItemInfo = SyncUtil.SyncableItemInfoFromJsonItemRef(remoteItemData["item"], remoteAuthors[i]); if (localAuthorItemInfo.Created.ReplicaId != remoteAuthorItemInfo.Created.ReplicaId) { return(DuplicateStatus.None); } if (localAuthorItemInfo.Created.ReplicaTickCount != remoteAuthorItemInfo.Created.ReplicaTickCount) { return(DuplicateStatus.None); } } return(DuplicateStatus.Exact); }
// // **** Methods for pulling sorted data // public IEnumerable <SortedBatch> PullSortedBatches() { // start the first batch and range SortedBatch pendingBatch = new SortedBatch(); long sizeOfBatch = 0; // intialize the range set build because we will be pullig // rows now and need to calculate the correct range sets BatchRangeSetBuilder rangeSetBuilder = new BatchRangeSetBuilder(_srcKnowledge.GetSyncIdFormatGroup().ItemIdFormat, _tablesInApplyOrder); rangeSetBuilder.StartBuildingFirstBatchRangeSet(); // for each table in apply order foreach (string tableName in _tablesInApplyOrder) { // start the next table range rangeSetBuilder.StartNextTable(tableName); // if we have a datatable for this name SortedTable curTable; if (_sortedTables.TryGetValue(tableName, out curTable) && curTable._schema != null) { // add the current table to the batch we are working // on DataTable curDataTable = curTable._schema.Clone(); pendingBatch.sortedDataSet.Tables.Add(curDataTable); curDataTable.BeginLoadData(); // if there are no rows in the table just add it to // the current dataset and move on SyncId maxIdInCurrentTable = null; // pull the rows in SyncId order foreach (KeyValuePair <SyncId, SortedRow> kvp in curTable._rows) { long curRowSize = SyncUtil.GetRowSizeForObjectArray(kvp.Value._rowValues); if (curRowSize > (_maxSortedBatchSizeInKB * 1024)) { // Note: This code is modified to throw a more specific exception. // If we end up merging this code with the provider, then the caller has to be tested to // make sure it works with the logic in the provider codebase. throw SyncServiceException.CreateInternalServerError( String.Format(Strings.RowExceedsConfiguredBatchSize, _maxSortedBatchSizeInKB, tableName, curRowSize)); } // fixme: if this row won't fit then return // the current batch if ((sizeOfBatch + curRowSize) > (_maxSortedBatchSizeInKB * 1024)) { // * done loading data curDataTable.EndLoadData(); // * add last sync id in batch if (maxIdInCurrentTable == null) { // we have not added any rows to the // current table so we should create a // dummy id in the current table for // the range maxIdInCurrentTable = rangeSetBuilder.MakeDummyFirstRowID(tableName); } rangeSetBuilder.AddSyncId(tableName, maxIdInCurrentTable); // start a new batch BatchRangeSet curRS = rangeSetBuilder.FinishBuildingBatchRangeSet(); pendingBatch.sortedDataSetKnowledge = curRS.ProjectOnKnowledge(_srcKnowledge); yield return(pendingBatch); // *** tricky // after yielding the current batch we // need to start a new one for the rest of // the rows in this table. // we must reset all the neede state and // this is tricky maxIdInCurrentTable = null; // start a new batch pendingBatch = new SortedBatch(); sizeOfBatch = 0; // start a new range after the current one rangeSetBuilder.StartBuildingBatchRangeSet(curRS); // add the current table to the batch we are working // on curDataTable = curTable._schema.Clone(); pendingBatch.sortedDataSet.Tables.Add(curDataTable); curDataTable.BeginLoadData(); } AddSortedRowToDataTable(curDataTable, kvp.Value); sizeOfBatch += curRowSize; maxIdInCurrentTable = kvp.Key; } curDataTable.EndLoadData(); } } // we should always be working on a batch { Debug.Assert(pendingBatch != null); BatchRangeSet curRS = rangeSetBuilder.FinishLastBatchRangeSet(); pendingBatch.sortedDataSetKnowledge = curRS.ProjectOnKnowledge(_srcKnowledge); } yield return(pendingBatch); }
private async Task PushChanges() { ReportProgressAndCheckCacellation(new SyncProgress { Stage = SyncStage.FindingLocalChanges, PercentComplete = 0, Message = "Finding local changes" }); var request = new JObject { { "sessionID", _remoteSessionId } }; var localKnowledge = _store.GenerateLocalKnowledge(); var changedItemInfos = _store.LocateChangedItems(_remoteKnowledge).ToList(); int totalChanges = changedItemInfos.Count(); request.Add(new JProperty("knowledge", SyncUtil.KnowledgeToJson(localKnowledge))); request.Add(new JProperty("changeCount", totalChanges)); await _transport.TransportAsync(SyncEndpoint.PutChanges, request); ReportProgressAndCheckCacellation(new SyncProgress { Stage = SyncStage.FindingLocalChanges, PercentComplete = 100, Message = String.Format("Found {0} local changes", totalChanges) }); ReportProgressAndCheckCacellation(new SyncProgress { Stage = SyncStage.UploadingLocalChanges, PercentComplete = 0, Message = String.Format("Uploading {0} local changes", totalChanges) }); int maxBatchCount = PushMaxBatchCount; int maxBatchSize = PushMaxBatchSize; long startTick = Environment.TickCount; int previousPercentComplete = -1; int i = 0; var batchArray = new JArray(); int batchSize = 0; foreach (ISyncableItemInfo syncItemInfo in changedItemInfos) { i++; int percentComplete = ((i * 100) / totalChanges); if (percentComplete != previousPercentComplete) { ReportProgressAndCheckCacellation(new SyncProgress { Stage = SyncStage.UploadingLocalChanges, PercentComplete = 100, Message = String.Format("Uploading local changes, {0}% complete ({1})", percentComplete, String.Format("Averaging {0}ms/item over {1} items", (Environment.TickCount - startTick) / i, i)) }); } previousPercentComplete = percentComplete; var builder = SyncUtil.JsonItemFromSyncableItemInfo(syncItemInfo); if (!syncItemInfo.Deleted) { _store.BuildItemData(syncItemInfo, builder); } var singleItemRequest = new JObject { { "changeNumber", i }, { "item", builder } }; batchSize += singleItemRequest.ToString().Length; batchArray.Add(singleItemRequest); if (i == totalChanges || (i % maxBatchCount) == 0 || batchSize >= maxBatchSize) { var batchRequest = new JObject { { "sessionID", _remoteSessionId }, { "batch", batchArray } }; await _transport.TransportAsync(SyncEndpoint.PutItemDataBatch, batchRequest); batchArray = new JArray(); batchSize = 0; } } ReportProgressAndCheckCacellation(new SyncProgress { Stage = SyncStage.UploadingLocalChanges, PercentComplete = 100, Message = String.Format("Uploaded {0} local changes", totalChanges) }); await ApplyChanges(totalChanges); }
/// <summary> /// Returns whether the original should be cancelled /// </summary> public bool DoSync(object target, params object[] args) { if (!Multiplayer.ShouldSync) { return(false); } // todo limit per specific target/argument //if (Utils.MillisNow - lastSendTime < minTime) // return true; LoggingByteWriter writer = new LoggingByteWriter(); MpContext context = writer.MpContext(); writer.Log.Node(ToString()); writer.WriteInt32(syncId); SyncUtil.WriteContext(this, writer); var map = context.map; void SyncObj(object obj, SyncType type, string debugInfo) { writer.Log.Enter(debugInfo); try { SyncSerialization.WriteSyncObject(writer, obj, type); } finally { writer.Log.Exit(); } if (type.contextMap && obj is Map contextMap) { map = contextMap; } if (context.map is Map newMap) { if (map != null && map != newMap) { throw new Exception($"{this}: map mismatch ({map?.uniqueID} and {newMap?.uniqueID})"); } map = newMap; } } if (targetTransformer != null) { SyncObj(targetTransformer.Writer.DynamicInvoke(target, target, args), targetTransformer.NetworkType, "Target (transformed)"); } else { WriteTarget(target, args, SyncObj); } for (int i = 0; i < argTypes.Length; i++) { if (argTransformers[i] == null) { SyncObj(args[i], argTypes[i], $"Arg {i} {argNames[i]}"); } } for (int i = 0; i < argTypes.Length; i++) { if (argTransformers[i] is { } trans) { SyncObj(trans.Writer.DynamicInvoke(args[i], target, args), trans.NetworkType, $"Arg {i} {argNames[i]} (transformed)"); } } int mapId = map?.uniqueID ?? ScheduledCommand.Global; writer.Log.Node("Map id: " + mapId); Multiplayer.WriterLog.AddCurrentNode(writer); Multiplayer.Client.SendCommand(CommandType.Sync, mapId, writer.ToArray()); lastSendTime = Utils.MillisNow; return(true); }
private IEnumerable <SyncConflict> CheckForDuplicates(IDbConnection connection) { var conflicts = new List <SyncConflict>(); var changedItemInfos = _store.LocateChangedItems(_remoteKnowledge).ToList(); foreach (string itemType in _store.GetItemTypes()) { IDbCommand getInsertedItemsCommand = connection.CreateCommand(); getInsertedItemsCommand.CommandText = String.Format( "SELECT ItemID, SyncStatus, ItemType, GlobalCreatedReplica, CreatedTickCount, GlobalModifiedReplica, ModifiedTickCount, ItemData FROM SyncItems WHERE SyncStatus={0} AND ItemType='{1}'", (int)SyncStatus.Insert, itemType); using (IDataReader reader = getInsertedItemsCommand.ExecuteReader()) { while (reader != null && reader.Read()) { long itemId = Convert.ToInt64(reader["ItemID"]); IReplicaInfo createdReplicaInfo = SessionDbHelper.ReplicaInfoFromDataReader(reader, "Created"); IReplicaInfo modifiedReplicaInfo = SessionDbHelper.ReplicaInfoFromDataReader(reader, "Modified"); ISyncableItemInfo remoteItemInfo = new SyncableItemInfo { ItemType = itemType, Created = createdReplicaInfo, Modified = modifiedReplicaInfo, Deleted = false }; var remoteItemData = JObject.Parse((string)reader["ItemData"]); foreach (var changedItemInfo in changedItemInfos) { if (changedItemInfo.ItemType != remoteItemInfo.ItemType) { continue; } if (SyncUtil.KnowledgeContains(_remoteKnowledge, changedItemInfo.Created)) { continue; } // Inserted here without remote knowledge, could be a dup var builder = SyncUtil.JsonItemFromSyncableItemInfo(changedItemInfo); _store.BuildItemData(changedItemInfo, builder); var localItemData = new JObject { { "item", builder } }; DuplicateStatus dupStatus = _store.GetDuplicateStatus(remoteItemInfo.ItemType, localItemData, remoteItemData); if (dupStatus == DuplicateStatus.Exact) { SessionDbHelper.ReplaceAllItemRefs(connection, _store, remoteItemInfo, changedItemInfo); long tickCount = _store.IncrementLocalRepilcaTickCount(); IReplicaInfo modifiedReplica = new ReplicaInfo { ReplicaId = _store.GetLocalReplicaId(), ReplicaTickCount = tickCount }; SessionDbHelper.ResolveItemNoData(connection, remoteItemInfo, SyncStatus.DeleteNonExisting, modifiedReplica); break; } if (dupStatus == DuplicateStatus.Possible) { // TODO: clean this up, this call does more than we need SessionDbHelper.ResolveItemNoData(connection, remoteItemInfo, SyncStatus.InsertConflict, remoteItemInfo.Modified); conflicts.Add(new SyncConflict(itemId, SyncStatus.InsertConflict, changedItemInfo, remoteItemInfo)); break; } } } } } return(conflicts); }
private IEnumerable <SyncConflict> LoadConflicts(IDbConnection connection) { var conflicts = new List <SyncConflict>(); IDbCommand getInsertedItemsCommand = connection.CreateCommand(); getInsertedItemsCommand.CommandText = String.Format( "SELECT ItemID, SyncStatus, ItemType, GlobalCreatedReplica, CreatedTickCount, GlobalModifiedReplica, ModifiedTickCount, ItemData FROM SyncItems WHERE SyncStatus NOT IN ({0},{1},{2},{3},{4},{5})", (int)SyncStatus.Insert, (int)SyncStatus.Update, (int)SyncStatus.Delete, (int)SyncStatus.DeleteNonExisting, (int)SyncStatus.MayBeNeeded, (int)SyncStatus.InsertConflict ); using (var reader = getInsertedItemsCommand.ExecuteReader()) { while (reader != null && reader.Read()) { var createdReplicaInfo = SessionDbHelper.ReplicaInfoFromDataReader(reader, "Created"); var modifiedReplicaInfo = SessionDbHelper.ReplicaInfoFromDataReader(reader, "Modified"); var itemType = (string)reader["ItemType"]; var remoteItemInfo = new SyncableItemInfo { ItemType = itemType, Created = createdReplicaInfo, Modified = modifiedReplicaInfo, Deleted = false }; var itemId = Convert.ToInt64(reader["ItemID"]); var status = (SyncStatus)reader["SyncStatus"]; ISyncableItemInfo localItemInfo = _store.LocateCurrentItemInfo(remoteItemInfo); if (status == SyncStatus.UpdateConflict) { // Check to see if the "conflict" is actually an exact same update var builder = SyncUtil.JsonItemFromSyncableItemInfo(localItemInfo); _store.BuildItemData(localItemInfo, builder); var localItemData = new JObject { { "item", builder } }; var remoteItemData = JObject.Parse((string)reader["ItemData"]); var dupStatus = _store.GetDuplicateStatus(remoteItemInfo.ItemType, localItemData, remoteItemData); if (dupStatus == DuplicateStatus.Exact) { var tickCount = _store.IncrementLocalRepilcaTickCount(); var modifiedReplica = new ReplicaInfo { ReplicaId = _store.GetLocalReplicaId(), ReplicaTickCount = tickCount }; SessionDbHelper.ResolveItemNoData(connection, remoteItemInfo, SyncStatus.Update, modifiedReplica); // TODO: Really should have an update status that just updates the modified repos without doing everything else, but this should work continue; } } conflicts.Add(new SyncConflict(itemId, status, localItemInfo, remoteItemInfo)); } } return(conflicts); }
private async Task <IEnumerable <SyncConflict> > PullChanges() { ReportProgressAndCheckCacellation(new SyncProgress { Stage = SyncStage.FindingRemoteChanges, PercentComplete = 0, Message = "Looking for remote changes" }); var request = new JObject { { "sessionID", _remoteSessionId } }; var localKnowledge = _store.GenerateLocalKnowledge().ToList(); request.Add(new JProperty("knowledge", SyncUtil.KnowledgeToJson(localKnowledge))); JObject response = await _transport.TransportAsync(SyncEndpoint.GetChanges, request); _remoteKnowledge = SyncUtil.KnowledgeFromJson(response["knowledge"]); var totalChanges = (int)response["totalChanges"]; ReportProgressAndCheckCacellation(new SyncProgress { Stage = SyncStage.FindingRemoteChanges, PercentComplete = 100, Message = String.Format("Found {0} remote changes", totalChanges) }); ReportProgressAndCheckCacellation(new SyncProgress { Stage = SyncStage.DownloadingRemoteChanges, PercentComplete = 0, Message = String.Format("Downloading {0} remote changes", totalChanges) }); using (var connection = _syncSessionDbConnectionProvider.GetSyncSessionDbConnection(_localSessionId)) { connection.ExecuteNonQuery("BEGIN"); SessionDbHelper.ClearSyncItems(connection); long startTick = Environment.TickCount; int previousPercentComplete = -1; for (int i = 1; i <= totalChanges;) { i += await SaveChangesBatch(connection, localKnowledge, i); int percentComplete = ((i * 100) / totalChanges); if (percentComplete != previousPercentComplete) { ReportProgressAndCheckCacellation(new SyncProgress { Stage = SyncStage.DownloadingRemoteChanges, PercentComplete = percentComplete, Message = String.Format("Downloading remote changes, {0}% complete ({1})", percentComplete, String.Format("Averaging {0}ms/item over {1} items", (Environment.TickCount - startTick) / i, i)) }); } previousPercentComplete = percentComplete; } connection.ExecuteNonQuery("COMMIT"); ReportProgressAndCheckCacellation(new SyncProgress { Stage = SyncStage.DownloadingRemoteChanges, PercentComplete = 100, Message = String.Format("Downloaded all {0} remote changes", totalChanges) }); ReportProgressAndCheckCacellation(new SyncProgress { Stage = SyncStage.CheckingForConflicts, PercentComplete = 0, Message = "Looking for conflicts" }); var conflicts = new List <SyncConflict>(); conflicts.AddRange(CheckForDuplicates(connection)); conflicts.AddRange(LoadConflicts(connection)); ReportProgressAndCheckCacellation(new SyncProgress { Stage = SyncStage.CheckingForConflicts, PercentComplete = 100, Message = String.Format("Found {0} conflicts", conflicts.Count) }); return(conflicts); } }
/// <summary> /// Sync data from GeniusDB and hosted in .144 which is the transfer DB /// </summary> /// <seealso cref="M:Spring.Scheduling.Quartz.QuartzJobObject.Execute(Quartz.JobExecutionContext)"/> protected override void ExecuteInternal(JobExecutionContext context) { var startTime = DateTime.UtcNow; var logEntity = new SCHEDULERLOG { STARTTIME = startTime }; var strInfo = new StringBuilder(); string settingFilePath = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, @"config\EJVMigration2.xml"); var settingManager = new XmlSettingManager(File.ReadAllText(settingFilePath)); // Set Connection string settingManager.Init(string.Empty);//初始化 mapping strInfo.AppendFormat("Source [Type: {0} Address: {1}]\n", settingManager.SourceDb.Type, settingManager.SourceDb.Conn); strInfo.AppendFormat("Destination [Type: {0} Address: {1}]\n", settingManager.DestinationDb.Type, settingManager.DestinationDb.Conn); var lastSyncTime = new DateTime(2017, 3, 17); var currentSyncTime = startTime.AddHours(8); try { using (var dataSync = new DataSynchronizer(settingManager, lastSyncTime, currentSyncTime)) { dataSync.PostTaskExecuted += (sender, e) => strInfo.AppendFormat ("Post sync task {0} is executed.\n", e.TaskName); dataSync.Init(); var sourceTable = settingManager.SourceTableName.Split(','); strInfo.Append(dataSync.SyncEachTableFromMaxMtime(sourceTable, "asset_last_chg_dt")); //strInfo.Append(dataSync.Sync(sourceTable)); } var endTime = DateTime.UtcNow; //Update Bond Info strInfo.AppendFormat("Update Bond Info en&cn start at {0}.\n", DateTime.UtcNow.ToGMT8String()); SyncUtil.UpdateBondInfo(lastSyncTime, currentSyncTime); strInfo.AppendFormat("Update Bond Info en&cn completed at {0}.\n ", DateTime.UtcNow.ToGMT8String()); //Rebuild Index strInfo.AppendFormat("Rebuild Index at {0}.\n", DateTime.UtcNow.ToGMT8String()); var result1 = SolrClient.RebuildIndex("full"); endTime = DateTime.UtcNow; strInfo.AppendFormat("Rebuild Index completed at {0}.\n Result: {1}\n", DateTime.UtcNow.ToGMT8String(), result1); logEntity.ENDTIME = endTime; logEntity.JobStatus = (result1 == "Success") ? JobStatus.Success : JobStatus.Fail; logEntity.RUNDETAIL = strInfo.ToString(); WriteLogEntity(logEntity); } catch (Exception exception) { logEntity.ENDTIME = DateTime.UtcNow; logEntity.JobStatus = JobStatus.Fail; logEntity.RUNDETAIL = strInfo + "\n" + exception; WriteLogEntity(logEntity); } }
static void Main(string[] args) { var waitForUser = true; var userLimit = 0; var userAccount = string.Empty; var ignoreChangeToken = false; if (args != null && args.Length > 0) { bool.TryParse(args[0], out waitForUser); if (args.Length > 1) { int.TryParse(args[1], out userLimit); } if (args.Length > 2) { userAccount = args[2]; } if (args.Length > 3) { bool.TryParse(args[3], out ignoreChangeToken); } } SyncUtil.JsonOpen(); try { var userProfileSync = new ProfileSync(); userProfileSync.Sync(new SyncOptions { UserLimit = userLimit, UserAccountFilter = userAccount, IgnoreChangeToken = ignoreChangeToken }); } catch (Exception ex) { Console.ForegroundColor = ConsoleColor.Black; Console.WriteLine("/*"); Console.ForegroundColor = ConsoleColor.Red; Console.WriteLine("An error occurred while running ProfileSync.\n"); Console.ResetColor(); Console.WriteLine(ex.ToString()); Console.ForegroundColor = ConsoleColor.Black; Console.WriteLine("*/"); Console.ResetColor(); } SyncUtil.JsonClose(); if (!waitForUser) { return; } Console.WriteLine("\nPress Enter to exit"); Console.ReadLine(); }
/// <summary> /// Execute the actual job. The job data map will already have been /// applied as object property values by execute. The contract is /// exactly the same as for the standard Quartz execute method. /// </summary> /// <seealso cref="M:Spring.Scheduling.Quartz.QuartzJobObject.Execute(Quartz.JobExecutionContext)"/> protected override void ExecuteInternal(JobExecutionContext context) { var startTime = DateTime.UtcNow; var logEntity = new SCHEDULERLOG { STARTTIME = startTime, JOBTYPE = JobType }; var strInfo = new StringBuilder(); string settingFilePath = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, @"config\File-To-IPP-sync.xml"); var settingManager = new XmlSettingManager(File.ReadAllText(settingFilePath)); // Set Connection string var destinationDbConn = string.Empty; settingManager.Init(destinationDbConn);//初始化 mapping strInfo.AppendFormat("<p>Source [Type: {0} Address: {1}]\n", settingManager.SourceDb.Type, settingManager.SourceDb.Conn); strInfo.AppendFormat("Destination [Type: {0} Address: {1}]</p>\n", settingManager.DestinationDb.Type, settingManager.DestinationDb.Conn); var from = getMaxDateTime(settingManager); var to = startTime.AddHours(8).AddHours(-settingManager.DeltaHours); strInfo.AppendFormat("<p>Max '{0}' of table '{1}' : {2}.</p>", settingManager.DateKeyColumn, settingManager.DateKeyTable, from); strInfo.AppendFormat("<p>Sync duration : {0} to : {1} .</p>", from, to); try { using (var dataSync = new DataSynchronizer(settingManager, from, to)) { dataSync.TableSynched += (sender, e) => strInfo.AppendFormat ("{0} rows have been synchronized from {1} view in CMAFileDB to {2} table in IPP DB.\n", e.NumOfRowsSynched, e.Source, e.Dest); dataSync.PostTaskExecuted += (sender, e) => strInfo.AppendFormat ("Post sync task {0} is executed.\n", e.TaskName); dataSync.Init(); strInfo.Append(dataSync.Sync(new[] { "GetNewInstitution", "GetNewFile" })); } strInfo.AppendFormat("{0} table(s) be synchronized.\n", settingManager.TableMappings.Count()); //Update File Topic strInfo.AppendFormat("<p>Update File Topic start at {0}.\n", DateTime.UtcNow.ToGMT8String()); var result = SyncUtil.UpdateFileTopic(from, to); strInfo.AppendFormat("Update File Topic completed at {0}.\n Result: {1}</p>", DateTime.UtcNow.ToGMT8String(), result); //Rebuild Index strInfo.AppendFormat("<p>Rebuild Index at {0}.\n", DateTime.UtcNow.ToGMT8String()); var result1 = SolrClient.RebuildIndex("full"); var endTime = DateTime.UtcNow; strInfo.AppendFormat("Rebuild Index completed at {0}.\n Result: {1}</p>", DateTime.UtcNow.ToGMT8String(), result); if (result1 != "Success") { strInfo.AppendFormat("<p style=\"color:red;\">Solr rebuild failed:<br />{0}</p>", result1); } logEntity.ENDTIME = endTime; logEntity.JobStatus = (result == "Success" && result1 == "Success") ? JobStatus.Success : JobStatus.Fail; logEntity.RUNDETAIL = strInfo.ToString(); WriteLogEntity(logEntity); } catch (Exception exception) { logEntity.ENDTIME = DateTime.UtcNow; logEntity.JobStatus = JobStatus.Fail; logEntity.RUNDETAIL = strInfo + "\n<b>Exception detail:</b>\n" + exception + "\n<p>No tables synchronized.</p>"; WriteLogEntity(logEntity); } }