/// <summary> /// Enumerate all internal changes, no batch mode /// </summary> internal async Task <(BatchInfo, DatabaseChangesSelected)> EnumerateChangesInternalAsync( SyncContext context, ScopeInfo scopeInfo, DmSet configTables, string batchDirectory, ConflictResolutionPolicy policy, ICollection <FilterClause> filters) { // create the in memory changes set var changesSet = new DmSet(SyncConfiguration.DMSET_NAME); // Create the batch info, in memory // No need to geneate a directory name, since we are in memory var batchInfo = new BatchInfo(true, batchDirectory); using (var connection = this.CreateConnection()) { // Open the connection await connection.OpenAsync(); using (var transaction = connection.BeginTransaction()) { try { // changes that will be returned as selected changes var changes = new DatabaseChangesSelected(); foreach (var tableDescription in configTables.Tables) { // if we are in upload stage, so check if table is not download only if (context.SyncWay == SyncWay.Upload && tableDescription.SyncDirection == SyncDirection.DownloadOnly) { continue; } // if we are in download stage, so check if table is not download only if (context.SyncWay == SyncWay.Download && tableDescription.SyncDirection == SyncDirection.UploadOnly) { continue; } var builder = this.GetDatabaseBuilder(tableDescription); var syncAdapter = builder.CreateSyncAdapter(connection, transaction); // raise before event context.SyncStage = SyncStage.TableChangesSelecting; // launch any interceptor await this.InterceptAsync(new TableChangesSelectingArgs(context, tableDescription.TableName, connection, transaction)); // selected changes for the current table var tableSelectedChanges = new TableChangesSelected { TableName = tableDescription.TableName }; // Get Command DbCommand selectIncrementalChangesCommand; DbCommandType dbCommandType; if (this.CanBeServerProvider && context.Parameters != null && context.Parameters.Count > 0 && filters != null && filters.Count > 0) { var tableFilters = filters .Where(f => f.TableName.Equals(tableDescription.TableName, StringComparison.InvariantCultureIgnoreCase)); if (tableFilters != null && tableFilters.Count() > 0) { dbCommandType = DbCommandType.SelectChangesWitFilters; selectIncrementalChangesCommand = syncAdapter.GetCommand(dbCommandType, tableFilters); if (selectIncrementalChangesCommand == null) { throw new Exception("Missing command 'SelectIncrementalChangesCommand'"); } syncAdapter.SetCommandParameters(dbCommandType, selectIncrementalChangesCommand, tableFilters); } else { dbCommandType = DbCommandType.SelectChanges; selectIncrementalChangesCommand = syncAdapter.GetCommand(dbCommandType); if (selectIncrementalChangesCommand == null) { throw new Exception("Missing command 'SelectIncrementalChangesCommand'"); } syncAdapter.SetCommandParameters(dbCommandType, selectIncrementalChangesCommand); } } else { dbCommandType = DbCommandType.SelectChanges; selectIncrementalChangesCommand = syncAdapter.GetCommand(dbCommandType); if (selectIncrementalChangesCommand == null) { throw new Exception("Missing command 'SelectIncrementalChangesCommand'"); } syncAdapter.SetCommandParameters(dbCommandType, selectIncrementalChangesCommand); } // Get a clone of the table with tracking columns var dmTableChanges = this.BuildChangesTable(tableDescription.TableName, configTables); SetSelectChangesCommonParameters(context, scopeInfo, selectIncrementalChangesCommand); // Set filter parameters if any if (this.CanBeServerProvider && context.Parameters != null && context.Parameters.Count > 0 && filters != null && filters.Count > 0) { var tableFilters = filters .Where(f => f.TableName.Equals(tableDescription.TableName, StringComparison.InvariantCultureIgnoreCase)).ToList(); if (tableFilters != null && tableFilters.Count > 0) { foreach (var filter in tableFilters) { var parameter = context.Parameters.FirstOrDefault(p => p.ColumnName.Equals(filter.ColumnName, StringComparison.InvariantCultureIgnoreCase) && p.TableName.Equals(filter.TableName, StringComparison.InvariantCultureIgnoreCase)); if (parameter != null) { DbManager.SetParameterValue(selectIncrementalChangesCommand, parameter.ColumnName, parameter.Value); } } } } this.AddTrackingColumns <int>(dmTableChanges, "sync_row_is_tombstone"); // Get the reader using (var dataReader = selectIncrementalChangesCommand.ExecuteReader()) { while (dataReader.Read()) { var dataRow = this.CreateRowFromReader(dataReader, dmTableChanges); //DmRow dataRow = dmTableChanges.NewRow(); // assuming the row is not inserted / modified var state = DmRowState.Unchanged; // get if the current row is inserted, modified, deleted state = this.GetStateFromDmRow(dataRow, scopeInfo); if (state != DmRowState.Deleted && state != DmRowState.Modified && state != DmRowState.Added) { continue; } // add row dmTableChanges.Rows.Add(dataRow); // acceptchanges before modifying dataRow.AcceptChanges(); tableSelectedChanges.TotalChanges++; // Set the correct state to be applied if (state == DmRowState.Deleted) { dataRow.Delete(); tableSelectedChanges.Deletes++; } else if (state == DmRowState.Added) { dataRow.SetAdded(); tableSelectedChanges.Inserts++; } else if (state == DmRowState.Modified) { dataRow.SetModified(); tableSelectedChanges.Updates++; } } // Since we dont need this column anymore, remove it this.RemoveTrackingColumns(dmTableChanges, "sync_row_is_tombstone"); // add it to the DmSet changesSet.Tables.Add(dmTableChanges); } // add the stats to global stats changes.TableChangesSelected.Add(tableSelectedChanges); // Progress & Interceptor context.SyncStage = SyncStage.TableChangesSelected; var args = new TableChangesSelectedArgs(context, tableSelectedChanges, connection, transaction); this.ReportProgress(context, args); await this.InterceptAsync(args); } transaction.Commit(); // generate the batchpartinfo batchInfo.GenerateBatchInfo(0, changesSet); // Create a new in-memory batch info with an the changes DmSet return(batchInfo, changes); } catch (Exception) { throw; } finally { if (connection != null && connection.State == ConnectionState.Open) { connection.Close(); } } } } }
internal virtual bool InternalNeedsToUpgrade(ScopeInfo scopeInfo) { var version = SyncVersion.EnsureVersion(scopeInfo.Version); return(version < SyncVersion.Current); }
/// <summary> /// Be sure all tables are ready and configured for sync /// the ScopeSet Configuration MUST be filled by the schema form Database /// </summary> public virtual async Task <SyncContext> EnsureDatabaseAsync(SyncContext context, ScopeInfo scopeInfo) { DbConnection connection = null; try { var configuration = GetCacheConfiguration(); // Event progress context.SyncStage = SyncStage.DatabaseApplying; DatabaseApplyingEventArgs beforeArgs = new DatabaseApplyingEventArgs(this.ProviderTypeName, context.SyncStage, configuration); this.TryRaiseProgressEvent(beforeArgs, this.DatabaseApplying); // if config has been editer by user in event, save again this.SetCacheConfiguration(configuration); // If scope exists and lastdatetime sync is present, so database exists // Check if we don't have an OverwriteConfiguration (if true, we force the check) if (scopeInfo.LastSync.HasValue && !beforeArgs.OverwriteConfiguration) { return(context); } StringBuilder script = new StringBuilder(); // Open the connection using (connection = this.CreateConnection()) { await connection.OpenAsync(); using (var transaction = connection.BeginTransaction()) { foreach (var dmTable in configuration) { var builder = GetDatabaseBuilder(dmTable); // adding filter this.AddFilters(configuration, dmTable, builder); context.SyncStage = SyncStage.DatabaseTableApplying; DatabaseTableApplyingEventArgs beforeTableArgs = new DatabaseTableApplyingEventArgs(this.ProviderTypeName, context.SyncStage, dmTable.TableName); this.TryRaiseProgressEvent(beforeTableArgs, this.DatabaseTableApplying); string currentScript = null; if (beforeArgs.GenerateScript) { currentScript = builder.ScriptTable(connection, transaction); currentScript += builder.ScriptForeignKeys(connection, transaction); script.Append(currentScript); } builder.Create(connection, transaction); builder.CreateForeignKeys(connection, transaction); context.SyncStage = SyncStage.DatabaseTableApplied; DatabaseTableAppliedEventArgs afterTableArgs = new DatabaseTableAppliedEventArgs(this.ProviderTypeName, context.SyncStage, dmTable.TableName, currentScript); this.TryRaiseProgressEvent(afterTableArgs, this.DatabaseTableApplied); } context.SyncStage = SyncStage.DatabaseApplied; var afterArgs = new DatabaseAppliedEventArgs(this.ProviderTypeName, context.SyncStage, script.ToString()); this.TryRaiseProgressEvent(afterArgs, this.DatabaseApplied); transaction.Commit(); } connection.Close(); return(context); } } catch (Exception ex) { throw new SyncException(ex, SyncStage.DatabaseApplying, this.ProviderTypeName); } finally { if (connection != null && connection.State != ConnectionState.Closed) { connection.Close(); } } }
/// <summary> /// Handle a conflict /// </summary> internal ChangeApplicationAction HandleConflict(SyncConflict conflict, ScopeInfo scope, long timestamp, out DmRow finalRow) { finalRow = null; // overwrite apply action if we handle it (ie : user wants to change the action) if (this.ConflictActionInvoker != null) { (ConflictApplyAction, finalRow) = this.ConflictActionInvoker(conflict, Connection, Transaction); } // Default behavior and an error occured if (ConflictApplyAction == ApplyAction.Rollback) { conflict.ErrorMessage = "Rollback action taken on conflict"; conflict.Type = ConflictType.ErrorsOccurred; return(ChangeApplicationAction.Rollback); } // Local provider wins, update metadata if (ConflictApplyAction == ApplyAction.Continue) { var isMergeAction = finalRow != null; var row = isMergeAction ? finalRow : conflict.LocalRow; // Conflict on a line that is not present on the datasource if (row == null) { return(ChangeApplicationAction.Continue); } if (row != null) { // if we have a merge action, we apply the row on the server if (isMergeAction) { this.ApplyUpdate(row, scope, true); // TODO : Différencier le timestamp de mise à jour ou de création using (var updateMetadataCommand = GetCommand(DbCommandType.UpdateMetadata)) { // Deriving Parameters this.SetCommandParameters(DbCommandType.UpdateMetadata, updateMetadataCommand); // apply local row, set scope.id to null becoz applied locally var rowsApplied = this.InsertOrUpdateMetadatas(updateMetadataCommand, conflict.LocalRow, null); if (!rowsApplied) { throw new Exception("No metadatas rows found, can't update the server side"); } } } finalRow = isMergeAction ? row : conflict.LocalRow; return(ChangeApplicationAction.Continue); } return(ChangeApplicationAction.Rollback); } // We gonna apply with force the line if (ConflictApplyAction == ApplyAction.RetryWithForceWrite) { if (conflict.RemoteRow == null) { // TODO : Should Raise an error ? return(ChangeApplicationAction.Rollback); } bool operationComplete = false; // create a localscope to override values var localScope = new ScopeInfo { Name = scope.Name, LastTimestamp = timestamp }; DbCommandType commandType = DbCommandType.InsertMetadata; switch (conflict.Type) { // Remote source has row, Local don't have the row, so insert it case ConflictType.RemoteUpdateLocalNoRow: case ConflictType.RemoteInsertLocalNoRow: operationComplete = this.ApplyInsert(conflict.RemoteRow, localScope, true); commandType = DbCommandType.InsertMetadata; break; // Conflict, but both have delete the row, so nothing to do case ConflictType.RemoteDeleteLocalDelete: case ConflictType.RemoteDeleteLocalNoRow: operationComplete = true; break; // The remote has delete the row, and local has insert or update it // So delete the local row case ConflictType.RemoteDeleteLocalUpdate: case ConflictType.RemoteDeleteLocalInsert: operationComplete = this.ApplyDelete(conflict.RemoteRow, localScope, true); commandType = DbCommandType.UpdateMetadata; break; // Remote insert and local delete, sor insert again on local // but tracking line exist, so make an update on metadata case ConflictType.RemoteInsertLocalDelete: case ConflictType.RemoteUpdateLocalDelete: operationComplete = this.ApplyInsert(conflict.RemoteRow, localScope, true); commandType = DbCommandType.UpdateMetadata; break; // Remote insert and local insert/ update, take the remote row and update the local row case ConflictType.RemoteUpdateLocalInsert: case ConflictType.RemoteUpdateLocalUpdate: case ConflictType.RemoteInsertLocalInsert: case ConflictType.RemoteInsertLocalUpdate: operationComplete = this.ApplyUpdate(conflict.RemoteRow, localScope, true); commandType = DbCommandType.UpdateMetadata; break; case ConflictType.RemoteCleanedupDeleteLocalUpdate: case ConflictType.ErrorsOccurred: return(ChangeApplicationAction.Rollback); } using (var metadataCommand = GetCommand(commandType)) { // Deriving Parameters this.SetCommandParameters(commandType, metadataCommand); // force applying client row, so apply scope.id (client scope here) var rowsApplied = this.InsertOrUpdateMetadatas(metadataCommand, conflict.RemoteRow, scope.Id); if (!rowsApplied) { throw new Exception("No metadatas rows found, can't update the server side"); } } finalRow = conflict.RemoteRow; //After a force update, there is a problem, so raise exception if (!operationComplete) { var ex = $"Can't force operation for applyType {applyType}"; Debug.WriteLine(ex); finalRow = null; return(ChangeApplicationAction.Continue); } // tableProgress.ChangesApplied += 1; return(ChangeApplicationAction.Continue); } return(ChangeApplicationAction.Rollback); }
/// <summary> /// Get changes from local database /// </summary> /// <returns></returns> public Task <(long ClientTimestamp, BatchInfo ClientBatchInfo, DatabaseChangesSelected ClientChangesSelected)> GetChangesAsync(ScopeInfo localScopeInfo = null, DbConnection connection = default, DbTransaction transaction = default, CancellationToken cancellationToken = default, IProgress <ProgressArgs> progress = null) => RunInTransactionAsync(SyncStage.ChangesSelecting, async(ctx, connection, transaction) =>
/// <summary> /// Launch a synchronization with the specified mode /// </summary> public async Task <SyncContext> SynchronizeAsync(SyncType syncType, CancellationToken cancellationToken, IProgress <ProgressArgs> progress = null) { // for view purpose, if needed if (this.LocalOrchestrator?.Provider != null) { this.LocalOrchestrator.Provider.Options = this.Options; } if (this.RemoteOrchestrator?.Provider != null) { this.RemoteOrchestrator.Provider.Options = this.Options; } var jsonConverter = new JsonConverter <SyncSet>(); // Lock sync to prevent multi call to sync at the same time LockSync(); // Context, used to back and forth data between servers var context = new SyncContext(Guid.NewGuid()) { // set start time StartTime = DateTime.UtcNow, // if any parameters, set in context Parameters = this.Parameters, // set sync type (Normal, Reinitialize, ReinitializeWithUpload) SyncType = syncType }; this.SessionState = SyncSessionState.Synchronizing; this.SessionStateChanged?.Invoke(this, this.SessionState); try { if (cancellationToken.IsCancellationRequested) { cancellationToken.ThrowIfCancellationRequested(); } ScopeInfo scope = null; // Starts sync by : // - Getting local config we have set by code // - Ensure local scope is created (table and values) (context, scope) = await this.LocalOrchestrator.EnsureScopeAsync (context, this.Setup.ScopeName, this.Options.ScopeInfoTableName, cancellationToken, progress); if (cancellationToken.IsCancellationRequested) { cancellationToken.ThrowIfCancellationRequested(); } // check if we already have a schema from local if (!string.IsNullOrEmpty(scope.Schema)) { this.Schema = Newtonsoft.Json.JsonConvert.DeserializeObject <SyncSet>(scope.Schema); //this.Schema = JsonSerializer.Deserialize<SyncSet>(scope.Schema, new JsonSerializerOptions { IgnoreReadOnlyProperties = true, IgnoreNullValues = true, MaxDepth = int.MaxValue }); } else { // FIRST call to server // Get the server scope info and server reference id to local scope // Be sure options / schema from client are passed if needed // Then the configuration with full schema var serverSchema = await this.RemoteOrchestrator.EnsureSchemaAsync( context, this.Setup, cancellationToken, remoteProgress); context = serverSchema.context; this.Schema = serverSchema.schema; } if (cancellationToken.IsCancellationRequested) { cancellationToken.ThrowIfCancellationRequested(); } // on local orchestrator, get local changes // Most probably the schema has changed, so we passed it again (coming from Server) // Don't need to pass again Options since we are not modifying it between server and client var clientChanges = await this.LocalOrchestrator.GetChangesAsync( context, this.Schema, scope, this.Options.BatchSize, this.Options.BatchDirectory, cancellationToken, progress); if (cancellationToken.IsCancellationRequested) { cancellationToken.ThrowIfCancellationRequested(); } // set context context = clientChanges.context; // Get if we need to get all rows from the datasource var fromScratch = scope.IsNewScope || context.SyncType == SyncType.Reinitialize || context.SyncType == SyncType.ReinitializeWithUpload; // IF is new and we have a snapshot directory, try to apply a snapshot if (fromScratch) { // Get snapshot files var serverSnapshotChanges = await this.RemoteOrchestrator.GetSnapshotAsync( context, scope, this.Schema, this.Options.SnapshotsDirectory, this.Options.BatchDirectory, cancellationToken, remoteProgress); if (serverSnapshotChanges.serverBatchInfo != null) { var hasChanges = await serverSnapshotChanges.serverBatchInfo.HasDataAsync(); if (hasChanges) { context = await this.LocalOrchestrator.ApplySnapshotAndGetChangesAsync(context, scope, this.Schema, serverSnapshotChanges.serverBatchInfo, clientChanges.clientTimestamp, serverSnapshotChanges.remoteClientTimestamp, this.Options.DisableConstraintsOnApplyChanges, this.Options.BatchSize, this.Options.BatchDirectory, this.Options.UseBulkOperations, this.Options.CleanMetadatas, this.Options.ScopeInfoTableName, cancellationToken, progress); } } } var serverChanges = await this.RemoteOrchestrator.ApplyThenGetChangesAsync( context, scope, this.Schema, clientChanges.clientBatchInfo, this.Options.DisableConstraintsOnApplyChanges, this.Options.UseBulkOperations, this.Options.CleanMetadatas, this.Options.CleanFolder, this.Options.BatchSize, this.Options.BatchDirectory, this.Options.ConflictResolutionPolicy, cancellationToken, remoteProgress); if (cancellationToken.IsCancellationRequested) { cancellationToken.ThrowIfCancellationRequested(); } // set context context = serverChanges.context; // Serialize schema to be able to save it in client db if (string.IsNullOrEmpty(scope.Schema)) { var schemaLight = Newtonsoft.Json.JsonConvert.SerializeObject(this.Schema); scope.Schema = schemaLight; } // Policy is always Server policy, so reverse this policy to get the client policy var clientPolicy = this.Options.ConflictResolutionPolicy == ConflictResolutionPolicy.ServerWins ? ConflictResolutionPolicy.ClientWins : ConflictResolutionPolicy.ServerWins; var localChanges = await this.LocalOrchestrator.ApplyChangesAsync( context, scope, this.Schema, serverChanges.serverBatchInfo, clientPolicy, clientChanges.clientTimestamp, serverChanges.remoteClientTimestamp, this.Options.DisableConstraintsOnApplyChanges, this.Options.UseBulkOperations, this.Options.CleanMetadatas, this.Options.ScopeInfoTableName, true, cancellationToken, progress); context.TotalChangesDownloaded += localChanges.clientChangesApplied.TotalAppliedChanges; context.TotalChangesUploaded += clientChanges.clientChangesSelected.TotalChangesSelected; context.TotalSyncErrors += localChanges.clientChangesApplied.TotalAppliedChangesFailed; if (cancellationToken.IsCancellationRequested) { cancellationToken.ThrowIfCancellationRequested(); } } catch (SyncException se) { Debug.WriteLine($"Sync Exception: {se.Message}. TypeName:{se.TypeName}."); throw; } catch (Exception ex) { Debug.WriteLine($"Unknwon Exception: {ex.Message}."); throw new SyncException(ex, SyncStage.None); } finally { // End the current session this.SessionState = SyncSessionState.Ready; this.SessionStateChanged?.Invoke(this, this.SessionState); // unlock sync since it's over UnlockSync(); } return(context); }
/// <summary> /// Try to apply changes on the server. /// Internally will call ApplyInsert / ApplyUpdate or ApplyDelete /// </summary> /// <param name="dmChanges">Changes from remote</param> /// <returns>every lines not updated on the server side</returns> internal int ApplyChanges(DmView dmChanges, ScopeInfo scope, List <SyncConflict> conflicts) { int appliedRows = 0; DbCommand dbCommand; foreach (var dmRow in dmChanges) { bool operationComplete = false; if (applyType == DmRowState.Added) { operationComplete = this.ApplyInsert(dmRow, scope, false); if (operationComplete) { using (dbCommand = this.GetCommand(DbCommandType.InsertMetadata)) { this.SetCommandParameters(DbCommandType.InsertMetadata, dbCommand); this.InsertOrUpdateMetadatas(dbCommand, dmRow, scope.Id); } } } else if (applyType == DmRowState.Modified) { operationComplete = this.ApplyUpdate(dmRow, scope, false); if (operationComplete) { using (dbCommand = this.GetCommand(DbCommandType.UpdateMetadata)) { this.SetCommandParameters(DbCommandType.UpdateMetadata, dbCommand); this.InsertOrUpdateMetadatas(dbCommand, dmRow, scope.Id); } } } else if (applyType == DmRowState.Deleted) { operationComplete = this.ApplyDelete(dmRow, scope, false); if (operationComplete) { using (dbCommand = this.GetCommand(DbCommandType.UpdateMetadata)) { this.SetCommandParameters(DbCommandType.UpdateMetadata, dbCommand); this.InsertOrUpdateMetadatas(dbCommand, dmRow, scope.Id); } } } if (operationComplete) { // if no pb, increment then go to next row appliedRows++; } else { // Generate a conflict and add it conflicts.Add(GetConflict(dmRow)); } } return(appliedRows); }
public ScopeArgs(SyncContext context, ScopeInfo scope, DbConnection connection, DbTransaction transaction) : base(context, connection, transaction) { this.ScopeInfo = scope; }
/// <summary> /// Write scope in the local data source /// </summary> public virtual async Task <SyncContext> WriteClientScopeAsync(SyncContext context, string scopeInfoTableName, ScopeInfo scope, DbConnection connection, DbTransaction transaction, CancellationToken cancellationToken, IProgress <ProgressArgs> progress = null) { var scopeBuilder = this.GetScopeBuilder(); var scopeInfoBuilder = scopeBuilder.CreateScopeInfoBuilder(scopeInfoTableName, connection, transaction); this.Orchestrator.logger.LogDebug(SyncEventsId.WriteClientScope, scope); await scopeInfoBuilder.InsertOrUpdateClientScopeInfoAsync(scope).ConfigureAwait(false); return(context); }
/// <summary> /// Apply changes internal method for one Insert or Update or Delete for every dbSyncAdapter /// </summary> internal ChangeApplicationAction ApplyChangesInternal(SyncContext context, DbConnection connection, DbTransaction transaction, ScopeInfo fromScope, BatchInfo changes, DmRowState applyType, ChangesApplied changesApplied) { ChangeApplicationAction changeApplicationAction = ChangeApplicationAction.Continue; var configuration = GetCacheConfiguration(); // for each adapters (Zero to End for Insert / Updates -- End to Zero for Deletes for (int i = 0; i < configuration.Count; i++) { // If we have a delete we must go from Up to Down, orthewise Dow to Up index var tableDescription = (applyType != DmRowState.Deleted ? configuration[i] : configuration[configuration.Count - i - 1]); // if we are in upload stage, so check if table is not download only if (context.SyncWay == SyncWay.Upload && tableDescription.SyncDirection == SyncDirection.DownloadOnly) { continue; } // if we are in download stage, so check if table is not download only if (context.SyncWay == SyncWay.Download && tableDescription.SyncDirection == SyncDirection.UploadOnly) { continue; } var builder = this.GetDatabaseBuilder(tableDescription); var syncAdapter = builder.CreateSyncAdapter(connection, transaction); syncAdapter.ConflictApplyAction = configuration.GetApplyAction(); // Set syncAdapter properties syncAdapter.applyType = applyType; // Get conflict handler resolver if (syncAdapter.ConflictActionInvoker == null && this.ApplyChangedFailed != null) { syncAdapter.ConflictActionInvoker = GetConflictAction; } if (changes.BatchPartsInfo != null && changes.BatchPartsInfo.Count > 0) { // getting the table to be applied // we may have multiple batch files, so we can have multipe dmTable with the same Name // We can say that dmTable may be contained in several files foreach (DmTable dmTablePart in changes.GetTable(tableDescription.TableName)) { if (dmTablePart == null || dmTablePart.Rows.Count == 0) { continue; } // check and filter var dmChangesView = new DmView(dmTablePart, (r) => r.RowState == applyType); if (dmChangesView.Count == 0) { dmChangesView.Dispose(); dmChangesView = null; continue; } // Conflicts occured when trying to apply rows List <SyncConflict> conflicts = new List <SyncConflict>(); // Raise event progress only if there are rows to be applied context.SyncStage = SyncStage.TableChangesApplying; var args = new TableChangesApplyingEventArgs(this.ProviderTypeName, context.SyncStage, tableDescription.TableName, applyType); this.TryRaiseProgressEvent(args, this.TableChangesApplying); int rowsApplied; // applying the bulkchanges command if (configuration.UseBulkOperations && this.SupportBulkOperations) { rowsApplied = syncAdapter.ApplyBulkChanges(dmChangesView, fromScope, conflicts); } else { rowsApplied = syncAdapter.ApplyChanges(dmChangesView, fromScope, conflicts); } // If conflicts occured // Eventuall, conflicts are resolved on server side. if (conflicts != null && conflicts.Count > 0) { foreach (var conflict in conflicts) { var scopeBuilder = this.GetScopeBuilder(); var scopeInfoBuilder = scopeBuilder.CreateScopeInfoBuilder(connection, transaction); var localTimeStamp = scopeInfoBuilder.GetLocalTimestamp(); changeApplicationAction = syncAdapter.HandleConflict(conflict, fromScope, localTimeStamp, out DmRow resolvedRow); if (changeApplicationAction == ChangeApplicationAction.Continue) { // row resolved if (resolvedRow != null) { rowsApplied++; } } else { context.TotalSyncErrors++; // TODO : Should we break at the first error ? return(ChangeApplicationAction.Rollback); } } } // Get all conflicts resolved context.TotalSyncConflicts = conflicts.Where(c => c.Type != ConflictType.ErrorsOccurred).Sum(c => 1); // Handle sync progress for this syncadapter (so this table) var changedFailed = dmChangesView.Count - rowsApplied; // raise SyncProgress Event var existAppliedChanges = changesApplied.TableChangesApplied.FirstOrDefault( sc => string.Equals(sc.TableName, tableDescription.TableName) && sc.State == applyType); if (existAppliedChanges == null) { existAppliedChanges = new TableChangesApplied { TableName = tableDescription.TableName, Applied = rowsApplied, Failed = changedFailed, State = applyType }; changesApplied.TableChangesApplied.Add(existAppliedChanges); } else { existAppliedChanges.Applied += rowsApplied; existAppliedChanges.Failed += changedFailed; } // Event progress context.SyncStage = SyncStage.TableChangesApplied; var progressEventArgs = new TableChangesAppliedEventArgs(this.ProviderTypeName, context.SyncStage, existAppliedChanges); this.TryRaiseProgressEvent(progressEventArgs, this.TableChangesApplied); } } // Dispose conflict handler resolver if (syncAdapter.ConflictActionInvoker != null) { syncAdapter.ConflictActionInvoker = null; } } return(ChangeApplicationAction.Continue); }
/// <summary> /// Apply changes : Insert / Updates Delete /// the fromScope is local client scope when this method is called from server /// the fromScope is server scope when this method is called from client /// </summary> public virtual async Task <(SyncContext, ChangesApplied)> ApplyChangesAsync(SyncContext context, ScopeInfo fromScope, BatchInfo changes) { ChangeApplicationAction changeApplicationAction; DbTransaction applyTransaction = null; ChangesApplied changesApplied = new ChangesApplied(); DbConnection connection = null; try { using (connection = this.CreateConnection()) { await connection.OpenAsync(); // Create a transaction applyTransaction = connection.BeginTransaction(); // ----------------------------------------------------- // 0) Check if we are in a reinit mode // ----------------------------------------------------- if (context.SyncWay == SyncWay.Download && context.SyncType != SyncType.Normal) { changeApplicationAction = this.ResetInternal(context, connection, applyTransaction, fromScope); // Rollback if (changeApplicationAction == ChangeApplicationAction.Rollback) { throw new SyncException("Rollback during reset tables", context.SyncStage, this.ProviderTypeName, SyncExceptionType.Rollback); } } // ----------------------------------------------------- // 1) Applying deletes. Do not apply deletes if we are in a new database // ----------------------------------------------------- if (!fromScope.IsNewScope) { changeApplicationAction = this.ApplyChangesInternal(context, connection, applyTransaction, fromScope, changes, DmRowState.Deleted, changesApplied); // Rollback if (changeApplicationAction == ChangeApplicationAction.Rollback) { throw new SyncException("Rollback during applying deletes", context.SyncStage, this.ProviderTypeName, SyncExceptionType.Rollback); } } // ----------------------------------------------------- // 1) Applying Inserts // ----------------------------------------------------- changeApplicationAction = this.ApplyChangesInternal(context, connection, applyTransaction, fromScope, changes, DmRowState.Added, changesApplied); // Rollback if (changeApplicationAction == ChangeApplicationAction.Rollback) { throw new SyncException("Rollback during applying inserts", context.SyncStage, this.ProviderTypeName, SyncExceptionType.Rollback); } // ----------------------------------------------------- // 1) Applying updates // ----------------------------------------------------- changeApplicationAction = this.ApplyChangesInternal(context, connection, applyTransaction, fromScope, changes, DmRowState.Modified, changesApplied); // Rollback if (changeApplicationAction == ChangeApplicationAction.Rollback) { throw new SyncException("Rollback during applying updates", context.SyncStage, this.ProviderTypeName, SyncExceptionType.Rollback); } applyTransaction.Commit(); return(context, changesApplied); } } catch (SyncException se) { throw; } catch (Exception ex) { throw new SyncException(ex, SyncStage.TableChangesApplying, this.ProviderTypeName); } finally { if (applyTransaction != null) { applyTransaction.Dispose(); applyTransaction = null; } if (connection != null && connection.State == ConnectionState.Open) { connection.Close(); } if (changes != null) { changes.Clear(); } } }
/// <summary> /// Here we are reseting all tables and tracking tables to be able to Reinitialize completely /// </summary> private ChangeApplicationAction ResetInternal(SyncContext context, DbConnection connection, DbTransaction transaction, ScopeInfo fromScope) { var configuration = GetCacheConfiguration(); for (int i = 0; i < configuration.Count; i++) { var tableDescription = configuration[configuration.Count - i - 1]; var builder = this.GetDatabaseBuilder(tableDescription); var syncAdapter = builder.CreateSyncAdapter(connection, transaction); // reset table syncAdapter.ResetTable(tableDescription); } return(ChangeApplicationAction.Continue); }
internal virtual async Task <ScopeInfo> InternalUpgradeAsync(SyncContext context, SyncSet schema, SyncSetup setup, ScopeInfo scopeInfo, DbScopeBuilder builder, DbConnection connection, DbTransaction transaction, CancellationToken cancellationToken, IProgress <ProgressArgs> progress) { var version = SyncVersion.EnsureVersion(scopeInfo.Version); var oldVersion = version.Clone() as Version; // beta version if (version.Major == 0) { if (version.Minor <= 5) { version = await AutoUpgdrateToNewVersionAsync(context, new Version(0, 6, 0), connection, transaction, cancellationToken, progress).ConfigureAwait(false); } if (version.Minor == 6 && version.Build == 0) { version = await UpgdrateTo601Async(context, schema, setup, connection, transaction, cancellationToken, progress).ConfigureAwait(false); } if (version.Minor == 6 && version.Build == 1) { version = await UpgdrateTo602Async(context, schema, setup, connection, transaction, cancellationToken, progress).ConfigureAwait(false); } if (version.Minor == 6 && version.Build >= 2) { version = await UpgdrateTo700Async(context, schema, setup, connection, transaction, cancellationToken, progress).ConfigureAwait(false); } if (version.Minor == 7 && version.Build == 0) { version = await AutoUpgdrateToNewVersionAsync(context, new Version(0, 7, 1), connection, transaction, cancellationToken, progress).ConfigureAwait(false); } if (version.Minor == 7 && version.Build == 1) { version = await AutoUpgdrateToNewVersionAsync(context, new Version(0, 7, 2), connection, transaction, cancellationToken, progress).ConfigureAwait(false); } if (version.Minor == 7 && version.Build == 2) { version = await AutoUpgdrateToNewVersionAsync(context, new Version(0, 7, 3), connection, transaction, cancellationToken, progress).ConfigureAwait(false); } if (version.Minor == 7 && version.Build >= 3) { version = await AutoUpgdrateToNewVersionAsync(context, new Version(0, 8, 0), connection, transaction, cancellationToken, progress).ConfigureAwait(false); } } if (oldVersion != version) { scopeInfo.Version = version.ToString(); scopeInfo = await this.InternalSaveScopeAsync(context, DbScopeType.Client, scopeInfo, builder, connection, transaction, cancellationToken, progress).ConfigureAwait(false); } return(scopeInfo); }
/// <summary> /// Enumerate all internal changes, no batch mode /// </summary> internal async Task <(BatchInfo, DatabaseChangesSelected)> EnumerateChangesInBatchesInternalAsync (SyncContext context, ScopeInfo scopeInfo, int downloadBatchSizeInKB, DmSet configTables, string batchDirectory, ConflictResolutionPolicy policy, ICollection <FilterClause> filters) { DmTable dmTable = null; // memory size total double memorySizeFromDmRows = 0L; var batchIndex = 0; // this batch info won't be in memory, it will be be batched var batchInfo = new BatchInfo(false, batchDirectory); // directory where all files will be stored batchInfo.GenerateNewDirectoryName(); // Create stats object to store changes count var changes = new DatabaseChangesSelected(); using (var connection = this.CreateConnection()) { try { // Open the connection await connection.OpenAsync(); using (var transaction = connection.BeginTransaction()) { // create the in memory changes set var changesSet = new DmSet(configTables.DmSetName); foreach (var tableDescription in configTables.Tables) { // if we are in upload stage, so check if table is not download only if (context.SyncWay == SyncWay.Upload && tableDescription.SyncDirection == SyncDirection.DownloadOnly) { continue; } // if we are in download stage, so check if table is not download only if (context.SyncWay == SyncWay.Download && tableDescription.SyncDirection == SyncDirection.UploadOnly) { continue; } var builder = this.GetDatabaseBuilder(tableDescription); var syncAdapter = builder.CreateSyncAdapter(connection, transaction); // raise before event context.SyncStage = SyncStage.TableChangesSelecting; var tableChangesSelectingArgs = new TableChangesSelectingArgs(context, tableDescription.TableName, connection, transaction); // launc interceptor if any await this.InterceptAsync(tableChangesSelectingArgs); // Get Command DbCommand selectIncrementalChangesCommand; DbCommandType dbCommandType; if (this.CanBeServerProvider && context.Parameters != null && context.Parameters.Count > 0 && filters != null && filters.Count > 0) { var tableFilters = filters .Where(f => f.TableName.Equals(tableDescription.TableName, StringComparison.InvariantCultureIgnoreCase)); if (tableFilters != null && tableFilters.Count() > 0) { dbCommandType = DbCommandType.SelectChangesWitFilters; selectIncrementalChangesCommand = syncAdapter.GetCommand(dbCommandType, tableFilters); if (selectIncrementalChangesCommand == null) { throw new Exception("Missing command 'SelectIncrementalChangesCommand' "); } syncAdapter.SetCommandParameters(dbCommandType, selectIncrementalChangesCommand, tableFilters); } else { dbCommandType = DbCommandType.SelectChanges; selectIncrementalChangesCommand = syncAdapter.GetCommand(dbCommandType); if (selectIncrementalChangesCommand == null) { throw new Exception("Missing command 'SelectIncrementalChangesCommand' "); } syncAdapter.SetCommandParameters(dbCommandType, selectIncrementalChangesCommand); } } else { dbCommandType = DbCommandType.SelectChanges; selectIncrementalChangesCommand = syncAdapter.GetCommand(dbCommandType); if (selectIncrementalChangesCommand == null) { throw new Exception("Missing command 'SelectIncrementalChangesCommand' "); } syncAdapter.SetCommandParameters(dbCommandType, selectIncrementalChangesCommand); } dmTable = this.BuildChangesTable(tableDescription.TableName, configTables); try { // Set commons parameters SetSelectChangesCommonParameters(context, scopeInfo, selectIncrementalChangesCommand); // Set filter parameters if any // Only on server side if (this.CanBeServerProvider && context.Parameters != null && context.Parameters.Count > 0 && filters != null && filters.Count > 0) { var filterTable = filters.Where(f => f.TableName.Equals(tableDescription.TableName, StringComparison.InvariantCultureIgnoreCase)).ToList(); if (filterTable != null && filterTable.Count > 0) { foreach (var filter in filterTable) { var parameter = context.Parameters.FirstOrDefault(p => p.ColumnName.Equals(filter.ColumnName, StringComparison.InvariantCultureIgnoreCase) && p.TableName.Equals(filter.TableName, StringComparison.InvariantCultureIgnoreCase)); if (parameter != null) { DbManager.SetParameterValue(selectIncrementalChangesCommand, parameter.ColumnName, parameter.Value); } } } } this.AddTrackingColumns <int>(dmTable, "sync_row_is_tombstone"); // Statistics var tableChangesSelected = new TableChangesSelected { TableName = tableDescription.TableName }; changes.TableChangesSelected.Add(tableChangesSelected); // Get the reader using (var dataReader = selectIncrementalChangesCommand.ExecuteReader()) { while (dataReader.Read()) { var dmRow = this.CreateRowFromReader(dataReader, dmTable); var state = DmRowState.Unchanged; state = this.GetStateFromDmRow(dmRow, scopeInfo); // If the row is not deleted inserted or modified, go next if (state != DmRowState.Deleted && state != DmRowState.Modified && state != DmRowState.Added) { continue; } var fieldsSize = DmTableSurrogate.GetRowSizeFromDataRow(dmRow); var dmRowSize = fieldsSize / 1024d; if (dmRowSize > downloadBatchSizeInKB) { var exc = $"Row is too big ({dmRowSize} kb.) for the current Configuration.DownloadBatchSizeInKB ({downloadBatchSizeInKB} kb.) Aborting Sync..."; throw new Exception(exc); } // Calculate the new memory size memorySizeFromDmRows = memorySizeFromDmRows + dmRowSize; // add row dmTable.Rows.Add(dmRow); tableChangesSelected.TotalChanges++; // acceptchanges before modifying dmRow.AcceptChanges(); // Set the correct state to be applied if (state == DmRowState.Deleted) { dmRow.Delete(); tableChangesSelected.Deletes++; } else if (state == DmRowState.Added) { dmRow.SetAdded(); tableChangesSelected.Inserts++; } else if (state == DmRowState.Modified) { dmRow.SetModified(); tableChangesSelected.Updates++; } // We exceed the memorySize, so we can add it to a batch if (memorySizeFromDmRows > downloadBatchSizeInKB) { // Since we dont need this column anymore, remove it this.RemoveTrackingColumns(dmTable, "sync_row_is_tombstone"); changesSet.Tables.Add(dmTable); // generate the batch part info batchInfo.GenerateBatchInfo(batchIndex, changesSet); // increment batch index batchIndex++; changesSet.Clear(); // Recreate an empty DmSet, then a dmTable clone changesSet = new DmSet(configTables.DmSetName); dmTable = dmTable.Clone(); this.AddTrackingColumns <int>(dmTable, "sync_row_is_tombstone"); // Init the row memory size memorySizeFromDmRows = 0L; // SyncProgress & interceptor context.SyncStage = SyncStage.TableChangesSelected; var loopTableChangesSelectedArgs = new TableChangesSelectedArgs(context, tableChangesSelected, connection, transaction); this.ReportProgress(context, loopTableChangesSelectedArgs); await this.InterceptAsync(loopTableChangesSelectedArgs); } } // Since we dont need this column anymore, remove it this.RemoveTrackingColumns(dmTable, "sync_row_is_tombstone"); context.SyncStage = SyncStage.TableChangesSelected; changesSet.Tables.Add(dmTable); // Init the row memory size memorySizeFromDmRows = 0L; // Event progress & interceptor context.SyncStage = SyncStage.TableChangesSelected; var tableChangesSelectedArgs = new TableChangesSelectedArgs(context, tableChangesSelected, connection, transaction); this.ReportProgress(context, tableChangesSelectedArgs); await this.InterceptAsync(tableChangesSelectedArgs); } } catch (Exception) { throw; } finally { } } // We are in batch mode, and we are at the last batchpart info if (changesSet != null && changesSet.HasTables && changesSet.HasChanges()) { var batchPartInfo = batchInfo.GenerateBatchInfo(batchIndex, changesSet); if (batchPartInfo != null) { batchPartInfo.IsLastBatch = true; } } transaction.Commit(); } } catch (Exception) { throw; } finally { if (connection != null && connection.State == ConnectionState.Open) { connection.Close(); } } } return(batchInfo, changes); }
/// <summary> /// Migrate an old setup configuration to a new one. This method is usefull if you are changing your SyncSetup when a database has been already configured previously /// </summary> public virtual async Task MigrationAsync(SyncSetup oldSetup, SyncSet schema, CancellationToken cancellationToken = default, IProgress <ProgressArgs> progress = null) { if (!this.StartTime.HasValue) { this.StartTime = DateTime.UtcNow; } // Get context or create a new one var ctx = this.GetContext(); using (var connection = this.Provider.CreateConnection()) { try { ctx.SyncStage = SyncStage.Migrating; // If schema does not have any table, just return if (schema == null || schema.Tables == null || !schema.HasTables) { throw new MissingTablesException(); } // Open connection await this.OpenConnectionAsync(connection, cancellationToken).ConfigureAwait(false); SyncProvision provision = SyncProvision.None; // Create a transaction using (var transaction = connection.BeginTransaction()) { await this.InterceptAsync(new TransactionOpenedArgs(ctx, connection, transaction), cancellationToken).ConfigureAwait(false); // Launch InterceptAsync on Migrating await this.InterceptAsync(new DatabaseMigratingArgs(ctx, schema, oldSetup, this.Setup, connection, transaction), cancellationToken).ConfigureAwait(false); this.logger.LogDebug(SyncEventsId.Migration, oldSetup); this.logger.LogDebug(SyncEventsId.Migration, this.Setup); // Migrate the db structure await this.Provider.MigrationAsync(ctx, schema, oldSetup, this.Setup, true, connection, transaction, cancellationToken, progress); // Now call the ProvisionAsync() to provision new tables provision = SyncProvision.Table | SyncProvision.TrackingTable | SyncProvision.StoredProcedures | SyncProvision.Triggers; await this.InterceptAsync(new DatabaseProvisioningArgs(ctx, provision, schema, connection, transaction), cancellationToken).ConfigureAwait(false); // Provision new tables if needed await this.Provider.ProvisionAsync(ctx, schema, this.Setup, provision, this.Options.ScopeInfoTableName, connection, transaction, cancellationToken, progress).ConfigureAwait(false); ScopeInfo localScope = null; ctx = await this.Provider.EnsureClientScopeAsync(ctx, this.Options.ScopeInfoTableName, connection, transaction, cancellationToken, progress).ConfigureAwait(false); (ctx, localScope) = await this.Provider.GetClientScopeAsync(ctx, this.Options.ScopeInfoTableName, this.ScopeName, connection, transaction, cancellationToken, progress).ConfigureAwait(false); localScope.Setup = this.Setup; localScope.Schema = schema; // Write scopes locally ctx = await this.Provider.WriteClientScopeAsync(ctx, this.Options.ScopeInfoTableName, localScope, connection, transaction, cancellationToken, progress).ConfigureAwait(false); await this.InterceptAsync(new TransactionCommitArgs(ctx, connection, transaction), cancellationToken).ConfigureAwait(false); transaction.Commit(); } ctx.SyncStage = SyncStage.Migrated; await this.CloseConnectionAsync(connection, cancellationToken).ConfigureAwait(false); var args = new DatabaseProvisionedArgs(ctx, provision, schema, connection); await this.InterceptAsync(args, cancellationToken).ConfigureAwait(false); this.ReportProgress(ctx, progress, args); // InterceptAsync Migrated var args2 = new DatabaseMigratedArgs(ctx, schema, this.Setup); await this.InterceptAsync(args2, cancellationToken).ConfigureAwait(false); this.ReportProgress(ctx, progress, args2); } catch (Exception ex) { RaiseError(ex); } finally { if (connection != null && connection.State == ConnectionState.Open) { connection.Close(); } } } }
/// <summary> /// Launch a synchronization with the specified mode /// </summary> public async Task <SyncContext> SynchronizeAsync(SyncType syncType, CancellationToken cancellationToken) { if (string.IsNullOrEmpty(this.scopeName)) { throw new Exception("Scope Name is mandatory"); } // Context, used to back and forth data between servers SyncContext context = new SyncContext(Guid.NewGuid()); // set start time context.StartTime = DateTime.Now; // if any parameters, set in context context.Parameters = this.Parameters; context.SyncType = syncType; this.SessionState = SyncSessionState.Synchronizing; this.SessionStateChanged?.Invoke(this, this.SessionState); ScopeInfo localScopeInfo = null, serverScopeInfo = null, localScopeReferenceInfo = null, scope = null; Guid fromId = Guid.Empty; long lastSyncTS = 0L; bool isNew = true; // Stats computed ChangesStatistics changesStatistics = new ChangesStatistics(); // tmp check error bool hasErrors = false; try { if (cancellationToken.IsCancellationRequested) { cancellationToken.ThrowIfCancellationRequested(); } // Setting the cancellation token this.LocalProvider.SetCancellationToken(cancellationToken); this.RemoteProvider.SetCancellationToken(cancellationToken); // Begin Session / Read the adapters context = await this.RemoteProvider.BeginSessionAsync(context); if (cancellationToken.IsCancellationRequested) { cancellationToken.ThrowIfCancellationRequested(); } context = await this.LocalProvider.BeginSessionAsync(context); if (cancellationToken.IsCancellationRequested) { cancellationToken.ThrowIfCancellationRequested(); } // ---------------------------------------- // 1) Read scope info // ---------------------------------------- // get the scope from local provider List <ScopeInfo> localScopes; List <ScopeInfo> serverScopes; (context, localScopes) = await this.LocalProvider.EnsureScopesAsync(context, scopeName); if (localScopes.Count != 1) { throw new Exception("On Local provider, we should have only one scope info"); } if (cancellationToken.IsCancellationRequested) { cancellationToken.ThrowIfCancellationRequested(); } localScopeInfo = localScopes[0]; (context, serverScopes) = await this.RemoteProvider.EnsureScopesAsync(context, scopeName, localScopeInfo.Id); if (serverScopes.Count != 2) { throw new Exception("On Remote provider, we should have two scopes (one for server and one for client side)"); } if (cancellationToken.IsCancellationRequested) { cancellationToken.ThrowIfCancellationRequested(); } serverScopeInfo = serverScopes.First(s => s.Id != localScopeInfo.Id); localScopeReferenceInfo = serverScopes.First(s => s.Id == localScopeInfo.Id); // ---------------------------------------- // 2) Build Configuration Object // ---------------------------------------- // Get Configuration from remote provider (context, this.Configuration) = await this.RemoteProvider.EnsureConfigurationAsync(context, this.Configuration); if (cancellationToken.IsCancellationRequested) { cancellationToken.ThrowIfCancellationRequested(); } // Invert policy on the client var configurationLocale = this.Configuration.Clone(); var policy = this.Configuration.ConflictResolutionPolicy; if (policy == ConflictResolutionPolicy.ServerWins) { configurationLocale.ConflictResolutionPolicy = ConflictResolutionPolicy.ClientWins; } if (policy == ConflictResolutionPolicy.ClientWins) { configurationLocale.ConflictResolutionPolicy = ConflictResolutionPolicy.ServerWins; } // Apply on local Provider SyncConfiguration configuration; (context, configuration) = await this.LocalProvider.EnsureConfigurationAsync(context, configurationLocale); if (cancellationToken.IsCancellationRequested) { cancellationToken.ThrowIfCancellationRequested(); } // ---------------------------------------- // 3) Ensure databases are ready // ---------------------------------------- // Server should have already the schema context = await this.RemoteProvider.EnsureDatabaseAsync(context, serverScopeInfo, DbBuilderOption.CreateOrUseExistingSchema | DbBuilderOption.CreateOrUseExistingTrackingTables); if (cancellationToken.IsCancellationRequested) { cancellationToken.ThrowIfCancellationRequested(); } // Client could have, or not, the tables context = await this.LocalProvider.EnsureDatabaseAsync(context, localScopeInfo, DbBuilderOption.CreateOrUseExistingSchema | DbBuilderOption.CreateOrUseExistingTrackingTables); if (cancellationToken.IsCancellationRequested) { cancellationToken.ThrowIfCancellationRequested(); } // ---------------------------------------- // 5) Get changes and apply them // ---------------------------------------- BatchInfo clientBatchInfo; BatchInfo serverBatchInfo; ChangesStatistics clientStatistics = null; ChangesStatistics serverStatistics = null; ChangesStatistics tmpClientStatistics = null; ChangesStatistics tmpServerStatistics = null; // fromId : not really needed on this case, since updated / inserted / deleted row has marked null // otherwise, lines updated by server or others clients are already syncked fromId = localScopeInfo.Id; // lastSyncTS : get lines inserted / updated / deteleted after the last sync commited lastSyncTS = localScopeInfo.LastTimestamp; // isNew : If isNew, lasttimestamp is not correct, so grab all isNew = localScopeInfo.IsNewScope; scope = new ScopeInfo { Id = fromId, IsNewScope = isNew, LastTimestamp = lastSyncTS }; (context, clientBatchInfo, clientStatistics) = await this.LocalProvider.GetChangeBatchAsync(context, scope); if (cancellationToken.IsCancellationRequested) { cancellationToken.ThrowIfCancellationRequested(); } // Apply on the Server Side // Since we are on the server, // we need to check the server client timestamp (not the client timestamp which is completely different) // fromId : When applying rows, make sure it's identified as applied by this client scope fromId = localScopeInfo.Id; // lastSyncTS : apply lines only if thye are not modified since last client sync lastSyncTS = localScopeReferenceInfo.LastTimestamp; // isNew : not needed isNew = false; scope = new ScopeInfo { Id = fromId, IsNewScope = isNew, LastTimestamp = lastSyncTS }; (context, serverStatistics) = await this.RemoteProvider.ApplyChangesAsync(context, scope, clientBatchInfo); if (cancellationToken.IsCancellationRequested) { cancellationToken.ThrowIfCancellationRequested(); } // Get changes from server // fromId : Make sure we don't select lines on server that has been already updated by the client fromId = localScopeInfo.Id; // lastSyncTS : apply lines only if thye are not modified since last client sync lastSyncTS = localScopeReferenceInfo.LastTimestamp; // isNew : make sure we take all lines if it's the first time we get isNew = localScopeReferenceInfo.IsNewScope; scope = new ScopeInfo { Id = fromId, IsNewScope = isNew, LastTimestamp = lastSyncTS }; (context, serverBatchInfo, tmpServerStatistics) = await this.RemoteProvider.GetChangeBatchAsync(context, scope); if (cancellationToken.IsCancellationRequested) { cancellationToken.ThrowIfCancellationRequested(); } // Update server stats if (serverStatistics == null) { serverStatistics = tmpServerStatistics; } else { clientStatistics.SelectedChanges = tmpServerStatistics.SelectedChanges; } // Apply local changes // fromId : When applying rows, make sure it's identified as applied by this server scope fromId = serverScopeInfo.Id; // lastSyncTS : apply lines only if they are not modified since last client sync lastSyncTS = localScopeInfo.LastTimestamp; // isNew : if IsNew, don't apply deleted rows from server isNew = localScopeInfo.IsNewScope; scope = new ScopeInfo { Id = fromId, IsNewScope = isNew, LastTimestamp = lastSyncTS }; (context, tmpClientStatistics) = await this.LocalProvider.ApplyChangesAsync(context, scope, serverBatchInfo); if (clientStatistics == null) { clientStatistics = tmpClientStatistics; } else { clientStatistics.AppliedChanges = tmpClientStatistics.AppliedChanges; } context.TotalChangesDownloaded = clientStatistics.TotalAppliedChanges; context.TotalChangesUploaded = serverStatistics.TotalAppliedChanges; context.TotalSyncErrors = clientStatistics.TotalAppliedChangesFailed; long serverTimestamp, clientTimestamp; if (cancellationToken.IsCancellationRequested) { cancellationToken.ThrowIfCancellationRequested(); } (context, serverTimestamp) = await this.RemoteProvider.GetLocalTimestampAsync(context); if (cancellationToken.IsCancellationRequested) { cancellationToken.ThrowIfCancellationRequested(); } (context, clientTimestamp) = await this.LocalProvider.GetLocalTimestampAsync(context); if (cancellationToken.IsCancellationRequested) { cancellationToken.ThrowIfCancellationRequested(); } context.CompleteTime = DateTime.Now; serverScopeInfo.IsNewScope = false; localScopeReferenceInfo.IsNewScope = false; localScopeInfo.IsNewScope = false; serverScopeInfo.LastSync = context.CompleteTime; localScopeReferenceInfo.LastSync = context.CompleteTime; localScopeInfo.LastSync = context.CompleteTime; serverScopeInfo.IsLocal = true; localScopeReferenceInfo.IsLocal = false; context = await this.RemoteProvider.WriteScopesAsync(context, new List <ScopeInfo> { serverScopeInfo, localScopeReferenceInfo }); serverScopeInfo.IsLocal = false; localScopeInfo.IsLocal = true; if (cancellationToken.IsCancellationRequested) { cancellationToken.ThrowIfCancellationRequested(); } context = await this.LocalProvider.WriteScopesAsync(context, new List <ScopeInfo> { localScopeInfo, serverScopeInfo }); if (cancellationToken.IsCancellationRequested) { cancellationToken.ThrowIfCancellationRequested(); } // Begin Session / Read the adapters context = await this.RemoteProvider.EndSessionAsync(context); context = await this.LocalProvider.EndSessionAsync(context); } catch (OperationCanceledException oce) { var error = SyncException.CreateOperationCanceledException(context.SyncStage, oce); HandleSyncError(error); hasErrors = true; throw; } catch (SyncException sex) { HandleSyncError(sex); hasErrors = true; throw; } catch (Exception ex) { var error = SyncException.CreateUnknowException(context.SyncStage, ex); HandleSyncError(error); hasErrors = true; throw; } finally { try { if (hasErrors) { // if EndSessionAsync() was never called, try a last time context = await this.RemoteProvider.EndSessionAsync(context); context = await this.LocalProvider.EndSessionAsync(context); } } catch (Exception) { // no raise } this.SessionState = SyncSessionState.Ready; this.SessionStateChanged?.Invoke(this, this.SessionState); } return(context); }
/// <summary> /// Get the local configuration, ensures the local scope is created /// </summary> /// <returns>current context, the local scope info created or get from the database and the configuration from the client if changed </returns> public async Task <ScopeInfo> GetClientScopeAsync(CancellationToken cancellationToken = default, IProgress <ProgressArgs> progress = null) { if (!this.StartTime.HasValue) { this.StartTime = DateTime.UtcNow; } // Get context or create a new one var ctx = this.GetContext(); ScopeInfo localScope = null; using (var connection = this.Provider.CreateConnection()) { try { ctx.SyncStage = SyncStage.ScopeLoading; // Open connection await this.OpenConnectionAsync(connection, cancellationToken).ConfigureAwait(false); using (var transaction = connection.BeginTransaction()) { await this.InterceptAsync(new TransactionOpenedArgs(ctx, connection, transaction), cancellationToken).ConfigureAwait(false); await this.InterceptAsync(new ScopeLoadingArgs(ctx, this.ScopeName, this.Options.ScopeInfoTableName, connection, transaction), cancellationToken).ConfigureAwait(false); ctx = await this.Provider.EnsureClientScopeAsync(ctx, this.Options.ScopeInfoTableName, connection, transaction, cancellationToken, progress).ConfigureAwait(false); (ctx, localScope) = await this.Provider.GetClientScopeAsync(ctx, this.Options.ScopeInfoTableName, this.ScopeName, connection, transaction, cancellationToken, progress).ConfigureAwait(false); await this.InterceptAsync(new TransactionCommitArgs(ctx, connection, transaction), cancellationToken).ConfigureAwait(false); transaction.Commit(); } ctx.SyncStage = SyncStage.ScopeLoaded; await this.CloseConnectionAsync(connection, cancellationToken).ConfigureAwait(false); var scopeArgs = new ScopeLoadedArgs(ctx, localScope, connection); await this.InterceptAsync(scopeArgs, cancellationToken).ConfigureAwait(false); this.ReportProgress(ctx, progress, scopeArgs); } catch (Exception ex) { RaiseError(ex); } finally { if (connection != null && connection.State == ConnectionState.Open) { connection.Close(); } } this.logger.LogInformation(SyncEventsId.GetClientScope, localScope); return(localScope); } }
/// <summary> /// Called when the sync ensure scopes are created /// </summary> public virtual async Task <(SyncContext, List <ScopeInfo>)> EnsureScopesAsync (SyncContext context, MessageEnsureScopes message) { DbConnection connection = null; try { context.SyncStage = SyncStage.ScopeLoading; List <ScopeInfo> scopes = new List <ScopeInfo>(); // Open the connection using (connection = this.CreateConnection()) { await connection.OpenAsync(); using (var transaction = connection.BeginTransaction()) { var scopeBuilder = this.GetScopeBuilder(); var scopeInfoBuilder = scopeBuilder.CreateScopeInfoBuilder( message.ScopeInfoTableName, connection, transaction); var needToCreateScopeInfoTable = scopeInfoBuilder.NeedToCreateScopeInfoTable(); // create the scope info table if needed if (needToCreateScopeInfoTable) { scopeInfoBuilder.CreateScopeInfoTable(); } // not the first time we ensure scopes, so get scopes if (!needToCreateScopeInfoTable) { // get all scopes shared by all (identified by scopeName) var lstScopes = scopeInfoBuilder.GetAllScopes(message.ScopeName); // try to get the scopes from database // could be two scopes if from server or a single scope if from client scopes = lstScopes.Where(s => (s.IsLocal == true || (message.ClientReferenceId.HasValue && s.Id == message.ClientReferenceId.Value))).ToList(); } // If no scope found, create it on the local provider if (scopes == null || scopes.Count <= 0) { scopes = new List <ScopeInfo>(); // create a new scope id for the current owner (could be server or client as well) var scope = new ScopeInfo(); scope.Id = Guid.NewGuid(); scope.Name = message.ScopeName; scope.IsLocal = true; scope.IsNewScope = true; scope.LastSync = null; scope = scopeInfoBuilder.InsertOrUpdateScopeInfo(scope); scopes.Add(scope); } else { //check if we have alread a good last sync. if no, treat it as new scopes.ForEach(sc => sc.IsNewScope = sc.LastSync == null); } // if we are not on the server, we have to check that we only have one scope if (!message.ClientReferenceId.HasValue && scopes.Count > 1) { throw new InvalidOperationException("On Local provider, we should have only one scope info"); } // if we have a reference in args, we need to get this specific line from database // this happen only on the server side if (message.ClientReferenceId.HasValue) { var refScope = scopes.FirstOrDefault(s => s.Id == message.ClientReferenceId); if (refScope == null) { refScope = new ScopeInfo(); refScope.Id = message.ClientReferenceId.Value; refScope.Name = message.ScopeName; refScope.IsLocal = false; refScope.IsNewScope = true; refScope.LastSync = null; refScope = scopeInfoBuilder.InsertOrUpdateScopeInfo(refScope); scopes.Add(refScope); } else { refScope.IsNewScope = refScope.LastSync == null; } } transaction.Commit(); } connection.Close(); } // Event progress this.TryRaiseProgressEvent( new ScopeEventArgs(this.ProviderTypeName, context.SyncStage, scopes.FirstOrDefault(s => s.IsLocal)), ScopeLoading); return(context, scopes); } catch (Exception ex) { throw new SyncException(ex, SyncStage.ScopeLoading, this.ProviderTypeName); } finally { if (connection != null && connection.State != ConnectionState.Closed) { connection.Close(); } } }
/// <summary> /// Launch apply bulk changes /// </summary> /// <returns></returns> public int ApplyBulkChanges(DmView dmChanges, ScopeInfo fromScope, List <SyncConflict> conflicts) { DbCommand bulkCommand = null; if (this.applyType == DmRowState.Added) { bulkCommand = this.GetCommand(DbCommandType.BulkInsertRows); this.SetCommandParameters(DbCommandType.BulkInsertRows, bulkCommand); } else if (this.applyType == DmRowState.Modified) { bulkCommand = this.GetCommand(DbCommandType.BulkUpdateRows); this.SetCommandParameters(DbCommandType.BulkUpdateRows, bulkCommand); } else if (this.applyType == DmRowState.Deleted) { bulkCommand = this.GetCommand(DbCommandType.BulkDeleteRows); this.SetCommandParameters(DbCommandType.BulkDeleteRows, bulkCommand); } else { throw new Exception("DmRowState not valid during ApplyBulkChanges operation"); } if (Transaction != null && Transaction.Connection != null) { bulkCommand.Transaction = Transaction; } //DmTable batchDmTable = dmChanges.Table.Clone(); DmTable failedDmtable = new DmTable { Culture = CultureInfo.InvariantCulture }; // Create the schema for failed rows (just add the Primary keys) this.AddSchemaForFailedRowsTable(failedDmtable); // Since the update and create timestamp come from remote, change name for the bulk operations var update_timestamp_column = dmChanges.Table.Columns["update_timestamp"].ColumnName; dmChanges.Table.Columns["update_timestamp"].ColumnName = "update_timestamp"; var create_timestamp_column = dmChanges.Table.Columns["create_timestamp"].ColumnName; dmChanges.Table.Columns["create_timestamp"].ColumnName = "create_timestamp"; // Make some parts of BATCH_SIZE for (int step = 0; step < dmChanges.Count; step += BATCH_SIZE) { // get upper bound max value var taken = step + BATCH_SIZE >= dmChanges.Count ? dmChanges.Count - step : BATCH_SIZE; using (var dmStepChanges = dmChanges.Take(step, taken)) { // execute the batch, through the provider ExecuteBatchCommand(bulkCommand, dmStepChanges, failedDmtable, fromScope); } } // Disposing command if (bulkCommand != null) { bulkCommand.Dispose(); bulkCommand = null; } // Since the update and create timestamp come from remote, change name for the bulk operations dmChanges.Table.Columns["update_timestamp"].ColumnName = update_timestamp_column; dmChanges.Table.Columns["create_timestamp"].ColumnName = create_timestamp_column; //foreach (var dmRow in dmChanges) //{ // // Cancel the delete state to be able to get the row, more simplier // if (applyType == DmRowState.Deleted) // dmRow.RejectChanges(); // // Load the datarow // DmRow dataRow = batchDmTable.LoadDataRow(dmRow.ItemArray, false); // // Apply the delete // // is it mandatory ? // if (applyType == DmRowState.Deleted) // dmRow.Delete(); // batchCount++; // rowCount++; // if (batchCount < BATCH_SIZE && rowCount < dmChanges.Count) // continue; // // Since the update and create timestamp come from remote, change name for the bulk operations // batchDmTable.Columns["update_timestamp"].ColumnName = "update_timestamp"; // batchDmTable.Columns["create_timestamp"].ColumnName = "create_timestamp"; // // execute the batch, through the provider // ExecuteBatchCommand(bulkCommand, batchDmTable, failedDmtable, fromScope); // // Clear the batch // batchDmTable.Clear(); // // Recreate a Clone // // TODO : Evaluate if it's necessary // batchDmTable = dmChanges.Table.Clone(); // batchCount = 0; //} // Update table progress //tableProgress.ChangesApplied = dmChanges.Count - failedDmtable.Rows.Count; if (failedDmtable.Rows.Count == 0) { return(dmChanges.Count); } // Check all conflicts raised var failedFilter = new Predicate <DmRow>(row => { if (row.RowState == DmRowState.Deleted) { return(failedDmtable.FindByKey(row.GetKeyValues(DmRowVersion.Original)) != null); } else { return(failedDmtable.FindByKey(row.GetKeyValues()) != null); } }); // New View var dmFailedRows = new DmView(dmChanges, failedFilter); // Generate a conflict and add it foreach (var dmFailedRow in dmFailedRows) { conflicts.Add(GetConflict(dmFailedRow)); } int failedRows = dmFailedRows.Count; // Dispose the failed view dmFailedRows.Dispose(); dmFailedRows = null; // return applied rows - failed rows (generating a conflict) return(dmChanges.Count - failedRows); }
ApplyThenGetChangesAsync(ScopeInfo clientScope, BatchInfo clientBatchInfo, CancellationToken cancellationToken = default, IProgress <ProgressArgs> progress = null) { if (!this.StartTime.HasValue) { this.StartTime = DateTime.UtcNow; } // Get context or create a new one var ctx = this.GetContext(); long remoteClientTimestamp = 0L; DatabaseChangesSelected serverChangesSelected = null; DatabaseChangesApplied clientChangesApplied = null; BatchInfo serverBatchInfo = null; SyncSet schema = null; using var connection = this.Provider.CreateConnection(); try { ctx.SyncStage = SyncStage.ChangesApplying; //Direction set to Upload ctx.SyncWay = SyncWay.Upload; // Open connection await this.OpenConnectionAsync(connection, cancellationToken).ConfigureAwait(false); DbTransaction transaction; // Create two transactions // First one to commit changes // Second one to get changes now that everything is commited using (transaction = connection.BeginTransaction()) { await this.InterceptAsync(new TransactionOpenedArgs(ctx, connection, transaction), cancellationToken).ConfigureAwait(false); // Maybe here, get the schema from server, issue from client scope name // Maybe then compare the schema version from client scope with schema version issued from server // Maybe if different, raise an error ? // Get scope if exists // Getting server scope assumes we have already created the schema on server var scopeBuilder = this.GetScopeBuilder(this.Options.ScopeInfoTableName); var serverScopeInfo = await this.InternalGetScopeAsync <ServerScopeInfo>(ctx, DbScopeType.Server, clientScope.Name, scopeBuilder, connection, transaction, cancellationToken, progress).ConfigureAwait(false); // Should we ? if (serverScopeInfo.Schema == null) { throw new MissingRemoteOrchestratorSchemaException(); } // Deserialiaze schema schema = serverScopeInfo.Schema; // Create message containing everything we need to apply on server side var applyChanges = new MessageApplyChanges(Guid.Empty, clientScope.Id, false, clientScope.LastServerSyncTimestamp, schema, this.Setup, this.Options.ConflictResolutionPolicy, this.Options.DisableConstraintsOnApplyChanges, this.Options.UseBulkOperations, this.Options.CleanMetadatas, this.Options.CleanFolder, clientBatchInfo); // Call provider to apply changes (ctx, clientChangesApplied) = await this.InternalApplyChangesAsync(ctx, applyChanges, connection, transaction, cancellationToken, progress).ConfigureAwait(false); await this.InterceptAsync(new TransactionCommitArgs(ctx, connection, transaction), cancellationToken).ConfigureAwait(false); // commit first transaction transaction.Commit(); } ctx.SyncStage = SyncStage.ChangesSelecting; ctx.ProgressPercentage = 0.55; using (transaction = connection.BeginTransaction()) { await this.InterceptAsync(new TransactionOpenedArgs(ctx, connection, transaction), cancellationToken).ConfigureAwait(false); //Direction set to Download ctx.SyncWay = SyncWay.Download; // JUST Before get changes, get the timestamp, to be sure to // get rows inserted / updated elsewhere since the sync is not over remoteClientTimestamp = await this.InternalGetLocalTimestampAsync(ctx, connection, transaction, cancellationToken, progress); // Get if we need to get all rows from the datasource var fromScratch = clientScope.IsNewScope || ctx.SyncType == SyncType.Reinitialize || ctx.SyncType == SyncType.ReinitializeWithUpload; var message = new MessageGetChangesBatch(clientScope.Id, Guid.Empty, fromScratch, clientScope.LastServerSyncTimestamp, schema, this.Setup, this.Options.BatchSize, this.Options.BatchDirectory); // Call interceptor await this.InterceptAsync(new DatabaseChangesSelectingArgs(ctx, message, connection, transaction), cancellationToken).ConfigureAwait(false); // When we get the chnages from server, we create the batches if it's requested by the client // the batch decision comes from batchsize from client (ctx, serverBatchInfo, serverChangesSelected) = await this.InternalGetChangesAsync(ctx, message, connection, transaction, cancellationToken, progress).ConfigureAwait(false); if (cancellationToken.IsCancellationRequested) { cancellationToken.ThrowIfCancellationRequested(); } // generate the new scope item this.CompleteTime = DateTime.UtcNow; var scopeHistory = new ServerHistoryScopeInfo { Id = clientScope.Id, Name = clientScope.Name, LastSyncTimestamp = remoteClientTimestamp, LastSync = this.CompleteTime, LastSyncDuration = this.CompleteTime.Value.Subtract(this.StartTime.Value).Ticks, }; // Write scopes locally var scopeBuilder = this.GetScopeBuilder(this.Options.ScopeInfoTableName); await this.InternalSaveScopeAsync(ctx, DbScopeType.ServerHistory, scopeHistory, scopeBuilder, connection, transaction, cancellationToken, progress).ConfigureAwait(false); // Commit second transaction for getting changes await this.InterceptAsync(new TransactionCommitArgs(ctx, connection, transaction), cancellationToken).ConfigureAwait(false); transaction.Commit(); } // Event progress & interceptor await this.CloseConnectionAsync(connection, cancellationToken).ConfigureAwait(false); } catch (Exception ex) { RaiseError(ex); } finally { await this.CloseConnectionAsync(connection, cancellationToken).ConfigureAwait(false); } return(remoteClientTimestamp, serverBatchInfo, this.Options.ConflictResolutionPolicy, clientChangesApplied, serverChangesSelected); }
/// <summary> /// Execute a batch command /// </summary> public abstract void ExecuteBatchCommand(DbCommand cmd, DmView applyTable, DmTable failedRows, ScopeInfo scope);
/// <summary> /// Get changes from remote database /// </summary> public virtual Task <(long RemoteClientTimestamp, BatchInfo ServerBatchInfo, DatabaseChangesSelected ServerChangesSelected)> GetChangesAsync(ScopeInfo clientScope, DbConnection connection = default, DbTransaction transaction = default, CancellationToken cancellationToken = default, IProgress <ProgressArgs> progress = null) => RunInTransactionAsync(SyncStage.ChangesSelecting, async(ctx, connection, transaction) =>
/// <summary> /// Handle a conflict /// The int returned is the conflict count I need /// </summary> internal async Task <(ChangeApplicationAction, int, DmRow)> HandleConflictAsync(DbSyncAdapter syncAdapter, SyncContext context, SyncConflict conflict, ConflictResolutionPolicy policy, ScopeInfo scope, long fromScopeLocalTimeStamp, DbConnection connection, DbTransaction transaction) { DmRow finalRow; ApplyAction conflictApplyAction; (conflictApplyAction, finalRow) = await this.GetConflictActionAsync(context, conflict, policy, connection, transaction); // Default behavior and an error occured if (conflictApplyAction == ApplyAction.Rollback) { conflict.ErrorMessage = "Rollback action taken on conflict"; conflict.Type = ConflictType.ErrorsOccurred; return(ChangeApplicationAction.Rollback, 0, null); } // Local provider wins, update metadata if (conflictApplyAction == ApplyAction.Continue) { var isMergeAction = finalRow != null; var row = isMergeAction ? finalRow : conflict.LocalRow; // Conflict on a line that is not present on the datasource if (row == null) { return(ChangeApplicationAction.Continue, 0, finalRow); } if (row != null) { // if we have a merge action, we apply the row on the server if (isMergeAction) { bool isUpdated = false; bool isInserted = false; // Insert metadata is a merge, actually var commandType = DbCommandType.UpdateMetadata; isUpdated = syncAdapter.ApplyUpdate(row, scope, true); if (!isUpdated) { // Insert the row isInserted = syncAdapter.ApplyInsert(row, scope, true); // Then update the row to mark this row as updated from server // and get it back to client isUpdated = syncAdapter.ApplyUpdate(row, scope, true); commandType = DbCommandType.InsertMetadata; } if (!isUpdated && !isInserted) { throw new Exception("Can't update the merge row."); } // IF we have insert the row in the server side, to resolve the conflict // Whe should update the metadatas correctly if (isUpdated || isInserted) { using (var metadataCommand = syncAdapter.GetCommand(commandType)) { // getting the row updated from server var dmTableRow = syncAdapter.GetRow(row); row = dmTableRow.Rows[0]; // Deriving Parameters syncAdapter.SetCommandParameters(commandType, metadataCommand); // Set the id parameter syncAdapter.SetColumnParametersValues(metadataCommand, row); var version = row.RowState == DmRowState.Deleted ? DmRowVersion.Original : DmRowVersion.Current; Guid?create_scope_id = row["create_scope_id"] != DBNull.Value ? (Guid?)row["create_scope_id"] : null; long createTimestamp = row["create_timestamp", version] != DBNull.Value ? Convert.ToInt64(row["create_timestamp", version]) : 0; // The trick is to force the row to be "created before last sync" // Even if we just inserted it // to be able to get the row in state Updated (and not Added) row["create_scope_id"] = create_scope_id; row["create_timestamp"] = fromScopeLocalTimeStamp - 1; // Update scope id is set to server side Guid?update_scope_id = row["update_scope_id"] != DBNull.Value ? (Guid?)row["update_scope_id"] : null; long updateTimestamp = row["update_timestamp", version] != DBNull.Value ? Convert.ToInt64(row["update_timestamp", version]) : 0; row["update_scope_id"] = null; row["update_timestamp"] = updateTimestamp; // apply local row, set scope.id to null becoz applied locally var rowsApplied = syncAdapter.InsertOrUpdateMetadatas(metadataCommand, row, null); if (!rowsApplied) { throw new Exception("No metadatas rows found, can't update the server side"); } } } } finalRow = isMergeAction ? row : conflict.LocalRow; // We don't do anything on the local provider, so we do not need to return a +1 on syncConflicts count return(ChangeApplicationAction.Continue, 0, finalRow); } return(ChangeApplicationAction.Rollback, 0, finalRow); } // We gonna apply with force the line if (conflictApplyAction == ApplyAction.RetryWithForceWrite) { if (conflict.RemoteRow == null) { // TODO : Should Raise an error ? return(ChangeApplicationAction.Rollback, 0, finalRow); } bool operationComplete = false; // create a localscope to override values var localScope = new ScopeInfo { Name = scope.Name, Timestamp = fromScopeLocalTimeStamp }; DbCommandType commandType = DbCommandType.InsertMetadata; bool needToUpdateMetadata = true; switch (conflict.Type) { // Remote source has row, Local don't have the row, so insert it case ConflictType.RemoteUpdateLocalNoRow: case ConflictType.RemoteInsertLocalNoRow: operationComplete = syncAdapter.ApplyInsert(conflict.RemoteRow, localScope, true); commandType = DbCommandType.InsertMetadata; break; // Conflict, but both have delete the row, so nothing to do case ConflictType.RemoteDeleteLocalDelete: case ConflictType.RemoteDeleteLocalNoRow: operationComplete = true; needToUpdateMetadata = false; break; // The remote has delete the row, and local has insert or update it // So delete the local row case ConflictType.RemoteDeleteLocalUpdate: case ConflictType.RemoteDeleteLocalInsert: operationComplete = syncAdapter.ApplyDelete(conflict.RemoteRow, localScope, true); commandType = DbCommandType.UpdateMetadata; break; // Remote insert and local delete, sor insert again on local // but tracking line exist, so make an update on metadata case ConflictType.RemoteInsertLocalDelete: case ConflictType.RemoteUpdateLocalDelete: operationComplete = syncAdapter.ApplyInsert(conflict.RemoteRow, localScope, true); commandType = DbCommandType.UpdateMetadata; break; // Remote insert and local insert/ update, take the remote row and update the local row case ConflictType.RemoteUpdateLocalInsert: case ConflictType.RemoteUpdateLocalUpdate: case ConflictType.RemoteInsertLocalInsert: case ConflictType.RemoteInsertLocalUpdate: operationComplete = syncAdapter.ApplyUpdate(conflict.RemoteRow, localScope, true); commandType = DbCommandType.UpdateMetadata; break; case ConflictType.RemoteCleanedupDeleteLocalUpdate: case ConflictType.ErrorsOccurred: return(ChangeApplicationAction.Rollback, 0, finalRow); } if (needToUpdateMetadata) { using (var metadataCommand = syncAdapter.GetCommand(commandType)) { // Deriving Parameters syncAdapter.SetCommandParameters(commandType, metadataCommand); // force applying client row, so apply scope.id (client scope here) var rowsApplied = syncAdapter.InsertOrUpdateMetadatas(metadataCommand, conflict.RemoteRow, scope.Id); if (!rowsApplied) { throw new Exception("No metadatas rows found, can't update the server side"); } } } finalRow = conflict.RemoteRow; //After a force update, there is a problem, so raise exception if (!operationComplete) { var ex = $"Can't force operation for applyType {syncAdapter.ApplyType}"; finalRow = null; return(ChangeApplicationAction.Continue, 0, finalRow); } // tableProgress.ChangesApplied += 1; return(ChangeApplicationAction.Continue, 1, finalRow); } return(ChangeApplicationAction.Rollback, 0, finalRow); }
/// <summary> /// Launch a synchronization with the specified mode /// </summary> public async Task <SyncContext> SynchronizeAsync(SyncType syncType, CancellationToken cancellationToken) { if (string.IsNullOrEmpty(this.scopeName)) { throw new ArgumentNullException("scopeName", "Scope Name is mandatory"); } // Context, used to back and forth data between servers SyncContext context = new SyncContext(Guid.NewGuid()) { // set start time StartTime = DateTime.Now, // if any parameters, set in context Parameters = this.Parameters, // set sync type (Normal, Reinitialize, ReinitializeWithUpload) SyncType = syncType }; this.SessionState = SyncSessionState.Synchronizing; this.SessionStateChanged?.Invoke(this, this.SessionState); ScopeInfo localScopeInfo = null, serverScopeInfo = null, localScopeReferenceInfo = null, scope = null; Guid fromId = Guid.Empty; long lastSyncTS = 0L; bool isNew = true; try { if (cancellationToken.IsCancellationRequested) { cancellationToken.ThrowIfCancellationRequested(); } // Setting the cancellation token this.LocalProvider.SetCancellationToken(cancellationToken); this.RemoteProvider.SetCancellationToken(cancellationToken); // Begin Session / Read the adapters context = await this.RemoteProvider.BeginSessionAsync(context); if (cancellationToken.IsCancellationRequested) { cancellationToken.ThrowIfCancellationRequested(); } context = await this.LocalProvider.BeginSessionAsync(context); if (cancellationToken.IsCancellationRequested) { cancellationToken.ThrowIfCancellationRequested(); } // ---------------------------------------- // 1) Read scope info // ---------------------------------------- // get the scope from local provider List <ScopeInfo> localScopes; List <ScopeInfo> serverScopes; (context, localScopes) = await this.LocalProvider.EnsureScopesAsync(context, scopeName); if (localScopes.Count != 1) { throw new Exception("On Local provider, we should have only one scope info"); } if (cancellationToken.IsCancellationRequested) { cancellationToken.ThrowIfCancellationRequested(); } localScopeInfo = localScopes[0]; (context, serverScopes) = await this.RemoteProvider.EnsureScopesAsync(context, scopeName, localScopeInfo.Id); if (serverScopes.Count != 2) { throw new Exception("On Remote provider, we should have two scopes (one for server and one for client side)"); } if (cancellationToken.IsCancellationRequested) { cancellationToken.ThrowIfCancellationRequested(); } serverScopeInfo = serverScopes.First(s => s.Id != localScopeInfo.Id); localScopeReferenceInfo = serverScopes.First(s => s.Id == localScopeInfo.Id); // ---------------------------------------- // 2) Build Configuration Object // ---------------------------------------- // Get Configuration from remote provider (context, this.Configuration) = await this.RemoteProvider.EnsureConfigurationAsync(context, this.Configuration); if (cancellationToken.IsCancellationRequested) { cancellationToken.ThrowIfCancellationRequested(); } // Invert policy on the client var configurationLocale = this.Configuration.Clone(); var policy = this.Configuration.ConflictResolutionPolicy; if (policy == ConflictResolutionPolicy.ServerWins) { configurationLocale.ConflictResolutionPolicy = ConflictResolutionPolicy.ClientWins; } if (policy == ConflictResolutionPolicy.ClientWins) { configurationLocale.ConflictResolutionPolicy = ConflictResolutionPolicy.ServerWins; } // Apply on local Provider SyncConfiguration configuration; (context, configuration) = await this.LocalProvider.EnsureConfigurationAsync(context, configurationLocale); if (cancellationToken.IsCancellationRequested) { cancellationToken.ThrowIfCancellationRequested(); } // ---------------------------------------- // 3) Ensure databases are ready // ---------------------------------------- // Server should have already the schema context = await this.RemoteProvider.EnsureDatabaseAsync(context, serverScopeInfo); if (cancellationToken.IsCancellationRequested) { cancellationToken.ThrowIfCancellationRequested(); } // Client could have, or not, the tables context = await this.LocalProvider.EnsureDatabaseAsync(context, localScopeInfo); if (cancellationToken.IsCancellationRequested) { cancellationToken.ThrowIfCancellationRequested(); } // ---------------------------------------- // 5) Get changes and apply them // ---------------------------------------- BatchInfo clientBatchInfo; BatchInfo serverBatchInfo; ChangesSelected clientChangesSelected = null; ChangesSelected serverChangesSelected = null; ChangesApplied clientChangesApplied = null; ChangesApplied serverChangesApplied = null; // fromId : not really needed on this case, since updated / inserted / deleted row has marked null // otherwise, lines updated by server or others clients are already syncked fromId = localScopeInfo.Id; // lastSyncTS : get lines inserted / updated / deteleted after the last sync commited lastSyncTS = localScopeInfo.LastTimestamp; // isNew : If isNew, lasttimestamp is not correct, so grab all isNew = localScopeInfo.IsNewScope; //Direction set to Upload context.SyncWay = SyncWay.Upload; scope = new ScopeInfo { Id = fromId, IsNewScope = isNew, LastTimestamp = lastSyncTS }; (context, clientBatchInfo, clientChangesSelected) = await this.LocalProvider.GetChangeBatchAsync(context, scope); if (cancellationToken.IsCancellationRequested) { cancellationToken.ThrowIfCancellationRequested(); } // Apply on the Server Side // Since we are on the server, // we need to check the server client timestamp (not the client timestamp which is completely different) // fromId : When applying rows, make sure it's identified as applied by this client scope fromId = localScopeInfo.Id; // lastSyncTS : apply lines only if thye are not modified since last client sync lastSyncTS = localScopeReferenceInfo.LastTimestamp; // isNew : not needed isNew = false; scope = new ScopeInfo { Id = fromId, IsNewScope = isNew, LastTimestamp = lastSyncTS }; (context, serverChangesApplied) = await this.RemoteProvider.ApplyChangesAsync(context, scope, clientBatchInfo); if (cancellationToken.IsCancellationRequested) { cancellationToken.ThrowIfCancellationRequested(); } // Get changes from server // get the archive if exists if (localScopeReferenceInfo.IsNewScope && !String.IsNullOrEmpty(this.Configuration.Archive)) { //// fromId : Make sure we don't select lines on server that has been already updated by the client //fromId = localScopeInfo.Id; //// lastSyncTS : apply lines only if thye are not modified since last client sync //lastSyncTS = localScopeReferenceInfo.LastTimestamp; //// isNew : make sure we take all lines if it's the first time we get //isNew = localScopeReferenceInfo.IsNewScope; //scope = new ScopeInfo { Id = fromId, IsNewScope = isNew, LastTimestamp = lastSyncTS }; ////Direction set to Download //context.SyncWay = SyncWay.Download; //(context, serverBatchInfo, serverChangesSelected) = await this.RemoteProvider.GetArchiveAsync(context, scope); //// fromId : When applying rows, make sure it's identified as applied by this server scope //fromId = serverScopeInfo.Id; //// lastSyncTS : apply lines only if they are not modified since last client sync //lastSyncTS = localScopeInfo.LastTimestamp; //// isNew : if IsNew, don't apply deleted rows from server //isNew = localScopeInfo.IsNewScope; //scope = new ScopeInfo { Id = fromId, IsNewScope = isNew, LastTimestamp = lastSyncTS }; //(context, clientChangesApplied) = await this.LocalProvider.ApplyArchiveAsync(context, scope, serverBatchInfo); //// Here we have to change the localScopeInfo.LastTimestamp to the good one //// last ts from archive //localScopeReferenceInfo.LastTimestamp = [something from the archive]; //// we are not new anymore //localScopeReferenceInfo.IsNewScope = false; } // fromId : Make sure we don't select lines on server that has been already updated by the client fromId = localScopeInfo.Id; // lastSyncTS : apply lines only if thye are not modified since last client sync lastSyncTS = localScopeReferenceInfo.LastTimestamp; // isNew : make sure we take all lines if it's the first time we get isNew = localScopeReferenceInfo.IsNewScope; scope = new ScopeInfo { Id = fromId, IsNewScope = isNew, LastTimestamp = lastSyncTS }; //Direction set to Download context.SyncWay = SyncWay.Download; (context, serverBatchInfo, serverChangesSelected) = await this.RemoteProvider.GetChangeBatchAsync(context, scope); if (cancellationToken.IsCancellationRequested) { cancellationToken.ThrowIfCancellationRequested(); } // Apply local changes // fromId : When applying rows, make sure it's identified as applied by this server scope fromId = serverScopeInfo.Id; // lastSyncTS : apply lines only if they are not modified since last client sync lastSyncTS = localScopeInfo.LastTimestamp; // isNew : if IsNew, don't apply deleted rows from server isNew = localScopeInfo.IsNewScope; scope = new ScopeInfo { Id = fromId, IsNewScope = isNew, LastTimestamp = lastSyncTS }; (context, clientChangesApplied) = await this.LocalProvider.ApplyChangesAsync(context, scope, serverBatchInfo); context.TotalChangesDownloaded = clientChangesApplied.TotalAppliedChanges; context.TotalChangesUploaded = clientChangesSelected.TotalChangesSelected; context.TotalSyncErrors = clientChangesApplied.TotalAppliedChangesFailed; long serverTimestamp, clientTimestamp; if (cancellationToken.IsCancellationRequested) { cancellationToken.ThrowIfCancellationRequested(); } (context, serverTimestamp) = await this.RemoteProvider.GetLocalTimestampAsync(context); if (cancellationToken.IsCancellationRequested) { cancellationToken.ThrowIfCancellationRequested(); } (context, clientTimestamp) = await this.LocalProvider.GetLocalTimestampAsync(context); if (cancellationToken.IsCancellationRequested) { cancellationToken.ThrowIfCancellationRequested(); } context.CompleteTime = DateTime.Now; serverScopeInfo.IsNewScope = false; localScopeReferenceInfo.IsNewScope = false; localScopeInfo.IsNewScope = false; serverScopeInfo.LastSync = context.CompleteTime; localScopeReferenceInfo.LastSync = context.CompleteTime; localScopeInfo.LastSync = context.CompleteTime; serverScopeInfo.IsLocal = true; localScopeReferenceInfo.IsLocal = false; context = await this.RemoteProvider.WriteScopesAsync(context, new List <ScopeInfo> { serverScopeInfo, localScopeReferenceInfo }); serverScopeInfo.IsLocal = false; localScopeInfo.IsLocal = true; if (cancellationToken.IsCancellationRequested) { cancellationToken.ThrowIfCancellationRequested(); } context = await this.LocalProvider.WriteScopesAsync(context, new List <ScopeInfo> { localScopeInfo, serverScopeInfo }); if (cancellationToken.IsCancellationRequested) { cancellationToken.ThrowIfCancellationRequested(); } } catch (SyncException se) { Debug.WriteLine($"Sync Exception: {se.Message}. Type:{se.Type}. On provider: {se.ProviderName}."); throw; } catch (Exception ex) { Debug.WriteLine($"Unknwon Exception: {ex.Message}."); throw new SyncException(ex, SyncStage.None, string.Empty); } finally { // End the current session context = await this.RemoteProvider.EndSessionAsync(context); context = await this.LocalProvider.EndSessionAsync(context); this.SessionState = SyncSessionState.Ready; this.SessionStateChanged?.Invoke(this, this.SessionState); } return(context); }
internal virtual async Task <ScopeInfo> InternalUpgradeAsync(SyncContext context, SyncSet schema, SyncSetup setup, ScopeInfo scopeInfo, DbScopeBuilder builder, DbConnection connection, DbTransaction transaction, CancellationToken cancellationToken, IProgress <ProgressArgs> progress) { var version = SyncVersion.EnsureVersion(scopeInfo.Version); var oldVersion = version.Clone() as Version; // beta version if (version.Major == 0) { if (version.Minor <= 5) { version = await UpgdrateTo600Async(context, schema, setup, connection, transaction, cancellationToken, progress).ConfigureAwait(false); } if (version.Minor == 6 && version.Build == 0) { version = await UpgdrateTo601Async(context, schema, setup, connection, transaction, cancellationToken, progress).ConfigureAwait(false); } if (version.Minor == 6 && version.Build == 1) { version = await UpgdrateTo602Async(context, schema, setup, connection, transaction, cancellationToken, progress).ConfigureAwait(false); } // last version of 0.6 Can be 0.6.2 or beta version 0.6.3 (that will never be released but still in the nuget packages available) if (version.Minor == 6 && version.Build >= 2) { version = await UpgdrateTo700Async(context, schema, setup, connection, transaction, cancellationToken, progress).ConfigureAwait(false); } if (version.Minor == 7 && version.Build == 0) { version = await UpgdrateTo701Async(context, schema, setup, connection, transaction, cancellationToken, progress).ConfigureAwait(false); } if (version.Minor == 7 && version.Build == 1) { version = await UpgdrateTo702Async(context, schema, setup, connection, transaction, cancellationToken, progress).ConfigureAwait(false); } if (version.Minor == 7 && version.Build == 2) { version = await UpgdrateTo703Async(context, schema, setup, connection, transaction, cancellationToken, progress).ConfigureAwait(false); } } if (oldVersion != version) { scopeInfo.Version = version.ToString(); scopeInfo = await this.InternalSaveScopeAsync(context, DbScopeType.Client, scopeInfo, builder, connection, transaction, cancellationToken, progress).ConfigureAwait(false); } return(scopeInfo); }
GetEstimatedChangesCountAsync(ScopeInfo localScopeInfo = null, CancellationToken cancellationToken = default, IProgress <ProgressArgs> progress = null) { if (!this.StartTime.HasValue) { this.StartTime = DateTime.UtcNow; } // Output long clientTimestamp = 0L; DatabaseChangesSelected clientChangesSelected = null; // Get context or create a new one var ctx = this.GetContext(); using (var connection = this.Provider.CreateConnection()) { try { ctx.SyncStage = SyncStage.ChangesSelecting; // Open connection await this.OpenConnectionAsync(connection, cancellationToken).ConfigureAwait(false); using (var transaction = connection.BeginTransaction()) { await this.InterceptAsync(new TransactionOpenedArgs(ctx, connection, transaction), cancellationToken).ConfigureAwait(false); // Get local scope, if not provided if (localScopeInfo == null) { ctx = await this.Provider.EnsureClientScopeAsync(ctx, this.Options.ScopeInfoTableName, connection, transaction, cancellationToken, progress).ConfigureAwait(false); (ctx, localScopeInfo) = await this.Provider.GetClientScopeAsync(ctx, this.Options.ScopeInfoTableName, this.ScopeName, connection, transaction, cancellationToken, progress).ConfigureAwait(false); } // If no schema in the client scope. Maybe the client scope table does not exists, or we never get the schema from server if (localScopeInfo.Schema == null) { throw new MissingLocalOrchestratorSchemaException(); } this.logger.LogInformation(SyncEventsId.GetClientScope, localScopeInfo); // On local, we don't want to chase rows from "others" // We just want our local rows, so we dont exclude any remote scope id, by setting scope id to NULL Guid?remoteScopeId = null; // lastSyncTS : get lines inserted / updated / deteleted after the last sync commited var lastSyncTS = localScopeInfo.LastSyncTimestamp; // isNew : If isNew, lasttimestamp is not correct, so grab all var isNew = localScopeInfo.IsNewScope; //Direction set to Upload ctx.SyncWay = SyncWay.Upload; // JUST before the whole process, get the timestamp, to be sure to // get rows inserted / updated elsewhere since the sync is not over clientTimestamp = await this.Provider.GetLocalTimestampAsync(ctx, connection, transaction, cancellationToken, progress); // Creating the message // Since it's an estimated count, we don't need to create batches, so we hard code the batchsize to 0 var message = new MessageGetChangesBatch(remoteScopeId, localScopeInfo.Id, isNew, lastSyncTS, localScopeInfo.Schema, this.Setup, 0, this.Options.BatchDirectory); // Call interceptor await this.InterceptAsync(new DatabaseChangesSelectingArgs(ctx, message, connection, transaction), cancellationToken).ConfigureAwait(false); this.logger.LogDebug(SyncEventsId.GetChanges, message); // Locally, if we are new, no need to get changes if (isNew) { clientChangesSelected = new DatabaseChangesSelected(); } else { (ctx, clientChangesSelected) = await this.Provider.GetEstimatedChangesCountAsync(ctx, message, connection, transaction, cancellationToken, progress).ConfigureAwait(false); } await this.InterceptAsync(new TransactionCommitArgs(ctx, connection, transaction), cancellationToken).ConfigureAwait(false); transaction.Commit(); } // Event progress & interceptor ctx.SyncStage = SyncStage.ChangesSelected; await this.CloseConnectionAsync(connection, cancellationToken).ConfigureAwait(false); } catch (Exception ex) { RaiseError(ex); } finally { if (connection != null && connection.State != ConnectionState.Closed) { connection.Close(); } } this.logger.LogInformation(SyncEventsId.GetChanges, new { ClientTimestamp = clientTimestamp, ClientChangesSelected = clientChangesSelected }); return(clientTimestamp, clientChangesSelected); } }
/// <summary> /// Here we are reseting all tables and tracking tables to be able to Reinitialize completely /// </summary> private ChangeApplicationAction ResetInternal(SyncContext context, DmSet configTables, DbConnection connection, DbTransaction transaction, ScopeInfo fromScope) { if (configTables == null || configTables.Tables.Count <= 0) { return(ChangeApplicationAction.Continue); } for (var i = 0; i < configTables.Tables.Count; i++) { var tableDescription = configTables.Tables[configTables.Tables.Count - i - 1]; var builder = this.GetDatabaseBuilder(tableDescription); var syncAdapter = builder.CreateSyncAdapter(connection, transaction); // reset table syncAdapter.ResetTable(tableDescription); } return(ChangeApplicationAction.Continue); }
ApplyChangesAsync(ScopeInfo scope, SyncSet schema, BatchInfo serverBatchInfo, long clientTimestamp, long remoteClientTimestamp, ConflictResolutionPolicy policy, CancellationToken cancellationToken = default, IProgress <ProgressArgs> progress = null) { if (!this.StartTime.HasValue) { this.StartTime = DateTime.UtcNow; } // Get context or create a new one var ctx = this.GetContext(); DatabaseChangesApplied clientChangesApplied = null; using (var connection = this.Provider.CreateConnection()) { try { ctx.SyncStage = SyncStage.ChangesApplying; // Open connection await this.OpenConnectionAsync(connection, cancellationToken).ConfigureAwait(false); // Create a transaction using (var transaction = connection.BeginTransaction()) { await this.InterceptAsync(new TransactionOpenedArgs(ctx, connection, transaction), cancellationToken).ConfigureAwait(false); // lastSyncTS : apply lines only if they are not modified since last client sync var lastSyncTS = scope.LastSyncTimestamp; // isNew : if IsNew, don't apply deleted rows from server var isNew = scope.IsNewScope; // We are in downloading mode ctx.SyncWay = SyncWay.Download; // Create the message containing everything needed to apply changes var applyChanges = new MessageApplyChanges(scope.Id, Guid.Empty, isNew, lastSyncTS, schema, this.Setup, policy, this.Options.DisableConstraintsOnApplyChanges, this.Options.UseBulkOperations, this.Options.CleanMetadatas, this.Options.CleanFolder, serverBatchInfo); // call interceptor await this.InterceptAsync(new DatabaseChangesApplyingArgs(ctx, applyChanges, connection, transaction), cancellationToken).ConfigureAwait(false); // Call apply changes on provider (ctx, clientChangesApplied) = await this.Provider.ApplyChangesAsync(ctx, applyChanges, connection, transaction, cancellationToken, progress).ConfigureAwait(false); if (cancellationToken.IsCancellationRequested) { cancellationToken.ThrowIfCancellationRequested(); } // check if we need to delete metadatas if (this.Options.CleanMetadatas && clientChangesApplied.TotalAppliedChanges > 0) { await this.Provider.DeleteMetadatasAsync(ctx, schema, this.Setup, lastSyncTS, connection, transaction, cancellationToken, progress); } // now the sync is complete, remember the time this.CompleteTime = DateTime.UtcNow; // generate the new scope item scope.IsNewScope = false; scope.LastSync = this.CompleteTime; scope.LastSyncTimestamp = clientTimestamp; scope.LastServerSyncTimestamp = remoteClientTimestamp; scope.LastSyncDuration = this.CompleteTime.Value.Subtract(this.StartTime.Value).Ticks; scope.Setup = this.Setup; // Write scopes locally ctx = await this.Provider.WriteClientScopeAsync(ctx, this.Options.ScopeInfoTableName, scope, connection, transaction, cancellationToken, progress).ConfigureAwait(false); await this.InterceptAsync(new TransactionCommitArgs(ctx, connection, transaction), cancellationToken).ConfigureAwait(false); transaction.Commit(); } ctx.SyncStage = SyncStage.ChangesApplied; await this.CloseConnectionAsync(connection, cancellationToken).ConfigureAwait(false); this.logger.LogInformation(SyncEventsId.ApplyChanges, clientChangesApplied); var databaseChangesAppliedArgs = new DatabaseChangesAppliedArgs(ctx, clientChangesApplied, connection); await this.InterceptAsync(databaseChangesAppliedArgs, cancellationToken).ConfigureAwait(false); this.ReportProgress(ctx, progress, databaseChangesAppliedArgs); } catch (Exception ex) { RaiseError(ex); } finally { if (connection != null && connection.State != ConnectionState.Closed) { connection.Close(); } } return(clientChangesApplied, scope); } }
/// <summary> /// Launch a synchronization with the specified mode /// </summary> public async Task <SyncContext> SynchronizeAsync(SyncType syncType, CancellationToken cancellationToken, IProgress <ProgressArgs> progress = null) { // Context, used to back and forth data between servers var context = new SyncContext(Guid.NewGuid()) { // set start time StartTime = DateTime.Now, // if any parameters, set in context Parameters = this.Parameters, // set sync type (Normal, Reinitialize, ReinitializeWithUpload) SyncType = syncType }; this.SessionState = SyncSessionState.Synchronizing; this.SessionStateChanged?.Invoke(this, this.SessionState); ScopeInfo localScopeInfo = null, serverScopeInfo = null, localScopeReferenceInfo = null, scope = null; var fromId = Guid.Empty; var lastSyncTS = 0L; var isNew = true; try { if (cancellationToken.IsCancellationRequested) { cancellationToken.ThrowIfCancellationRequested(); } // Setting the cancellation token this.LocalProvider.SetCancellationToken(cancellationToken); this.RemoteProvider.SetCancellationToken(cancellationToken); // Setting progress this.LocalProvider.SetProgress(progress); // ---------------------------------------- // 0) Begin Session / Get the Configuration from remote provider // If the configuration object is provided by the client, the server will be updated with it. // ---------------------------------------- (context, this.LocalProvider.Configuration) = await this.RemoteProvider.BeginSessionAsync(context, new MessageBeginSession { Configuration = this.LocalProvider.Configuration }); if (cancellationToken.IsCancellationRequested) { cancellationToken.ThrowIfCancellationRequested(); } // Locally, nothing really special. Eventually, editing the config object (context, this.LocalProvider.Configuration) = await this.LocalProvider.BeginSessionAsync(context, new MessageBeginSession { Configuration = this.LocalProvider.Configuration }); if (cancellationToken.IsCancellationRequested) { cancellationToken.ThrowIfCancellationRequested(); } // ---------------------------------------- // 1) Read scope info // ---------------------------------------- // get the scope from local provider List <ScopeInfo> localScopes; List <ScopeInfo> serverScopes; (context, localScopes) = await this.LocalProvider.EnsureScopesAsync(context, new MessageEnsureScopes { ScopeInfoTableName = this.LocalProvider.Configuration.ScopeInfoTableName, ScopeName = this.LocalProvider.Configuration.ScopeName, SerializationFormat = this.LocalProvider.Configuration.SerializationFormat }); if (localScopes.Count != 1) { throw new Exception("On Local provider, we should have only one scope info"); } if (cancellationToken.IsCancellationRequested) { cancellationToken.ThrowIfCancellationRequested(); } localScopeInfo = localScopes[0]; (context, serverScopes) = await this.RemoteProvider.EnsureScopesAsync(context, new MessageEnsureScopes { ScopeInfoTableName = this.LocalProvider.Configuration.ScopeInfoTableName, ScopeName = this.LocalProvider.Configuration.ScopeName, ClientReferenceId = localScopeInfo.Id, SerializationFormat = this.LocalProvider.Configuration.SerializationFormat }); if (serverScopes.Count != 2) { throw new Exception("On Remote provider, we should have two scopes (one for server and one for client side)"); } if (cancellationToken.IsCancellationRequested) { cancellationToken.ThrowIfCancellationRequested(); } serverScopeInfo = serverScopes.First(s => s.Id != localScopeInfo.Id); localScopeReferenceInfo = serverScopes.First(s => s.Id == localScopeInfo.Id); // ---------------------------------------- // 2) Build Configuration Object // ---------------------------------------- // Get Schema from remote provider (context, this.LocalProvider.Configuration.Schema) = await this.RemoteProvider.EnsureSchemaAsync(context, new MessageEnsureSchema { Schema = this.LocalProvider.Configuration.Schema, SerializationFormat = this.LocalProvider.Configuration.SerializationFormat }); if (cancellationToken.IsCancellationRequested) { cancellationToken.ThrowIfCancellationRequested(); } // Apply on local Provider (context, this.LocalProvider.Configuration.Schema) = await this.LocalProvider.EnsureSchemaAsync(context, new MessageEnsureSchema { Schema = this.LocalProvider.Configuration.Schema, SerializationFormat = this.LocalProvider.Configuration.SerializationFormat }); if (cancellationToken.IsCancellationRequested) { cancellationToken.ThrowIfCancellationRequested(); } // ---------------------------------------- // 3) Ensure databases are ready // ---------------------------------------- // Server should have already the schema context = await this.RemoteProvider.EnsureDatabaseAsync(context, new MessageEnsureDatabase { ScopeInfo = serverScopeInfo, Schema = this.LocalProvider.Configuration.Schema, Filters = this.LocalProvider.Configuration.Filters, SerializationFormat = this.LocalProvider.Configuration.SerializationFormat }); if (cancellationToken.IsCancellationRequested) { cancellationToken.ThrowIfCancellationRequested(); } // Client could have, or not, the tables context = await this.LocalProvider.EnsureDatabaseAsync(context, new MessageEnsureDatabase { ScopeInfo = localScopeInfo, Schema = this.LocalProvider.Configuration.Schema, Filters = this.LocalProvider.Configuration.Filters, SerializationFormat = this.LocalProvider.Configuration.SerializationFormat }); if (cancellationToken.IsCancellationRequested) { cancellationToken.ThrowIfCancellationRequested(); } // ---------------------------------------- // 5) Get changes and apply them // ---------------------------------------- BatchInfo clientBatchInfo; BatchInfo serverBatchInfo; DatabaseChangesSelected clientChangesSelected = null; DatabaseChangesSelected serverChangesSelected = null; DatabaseChangesApplied clientChangesApplied = null; DatabaseChangesApplied serverChangesApplied = null; // those timestamps will be registered as the "timestamp just before launch the sync" long serverTimestamp, clientTimestamp; if (cancellationToken.IsCancellationRequested) { cancellationToken.ThrowIfCancellationRequested(); } // Apply on the Server Side // Since we are on the server, // we need to check the server client timestamp (not the client timestamp which is completely different) var serverPolicy = this.LocalProvider.Configuration.ConflictResolutionPolicy; var clientPolicy = serverPolicy == ConflictResolutionPolicy.ServerWins ? ConflictResolutionPolicy.ClientWins : ConflictResolutionPolicy.ServerWins; // We get from local provider all rows not last updated from the server fromId = serverScopeInfo.Id; // lastSyncTS : get lines inserted / updated / deteleted after the last sync commited lastSyncTS = localScopeInfo.LastSyncTimestamp; // isNew : If isNew, lasttimestamp is not correct, so grab all isNew = localScopeInfo.IsNewScope; //Direction set to Upload context.SyncWay = SyncWay.Upload; // JUST before the whole process, get the timestamp, to be sure to // get rows inserted / updated elsewhere since the sync is not over (context, clientTimestamp) = await this.LocalProvider.GetLocalTimestampAsync(context, new MessageTimestamp { ScopeInfoTableName = this.LocalProvider.Configuration.ScopeInfoTableName, SerializationFormat = this.LocalProvider.Configuration.SerializationFormat }); if (cancellationToken.IsCancellationRequested) { cancellationToken.ThrowIfCancellationRequested(); } scope = new ScopeInfo { Id = fromId, IsNewScope = isNew, Timestamp = lastSyncTS }; (context, clientBatchInfo, clientChangesSelected) = await this.LocalProvider.GetChangeBatchAsync(context, new MessageGetChangesBatch { ScopeInfo = scope, Schema = this.LocalProvider.Configuration.Schema, Policy = clientPolicy, Filters = this.LocalProvider.Configuration.Filters, SerializationFormat = this.LocalProvider.Configuration.SerializationFormat }); if (cancellationToken.IsCancellationRequested) { cancellationToken.ThrowIfCancellationRequested(); } // fromId : When applying rows, make sure it's identified as applied by this client scope fromId = localScopeInfo.Id; // lastSyncTS : apply lines only if thye are not modified since last client sync lastSyncTS = localScopeReferenceInfo.LastSyncTimestamp; // isNew : not needed isNew = false; scope = new ScopeInfo { Id = fromId, IsNewScope = isNew, Timestamp = lastSyncTS }; (context, serverChangesApplied) = await this.RemoteProvider.ApplyChangesAsync(context, new MessageApplyChanges { FromScope = scope, Schema = this.LocalProvider.Configuration.Schema, Policy = serverPolicy, ScopeInfoTableName = this.LocalProvider.Configuration.ScopeInfoTableName, Changes = clientBatchInfo, SerializationFormat = this.LocalProvider.Configuration.SerializationFormat }); // if ConflictResolutionPolicy.ClientWins or Handler set to Client wins // Conflict occurs here and server loose. // Conflicts count should be temp saved because applychanges on client side won't raise any conflicts (and so property Context.TotalSyncConflicts will be reset to 0) var conflictsOnRemoteCount = context.TotalSyncConflicts; if (cancellationToken.IsCancellationRequested) { cancellationToken.ThrowIfCancellationRequested(); } // Get changes from server // fromId : Make sure we don't select lines on server that has been already updated by the client fromId = localScopeInfo.Id; // lastSyncTS : apply lines only if thye are not modified since last client sync lastSyncTS = localScopeReferenceInfo.LastSyncTimestamp; // isNew : make sure we take all lines if it's the first time we get isNew = localScopeReferenceInfo.IsNewScope; scope = new ScopeInfo { Id = fromId, IsNewScope = isNew, Timestamp = lastSyncTS }; //Direction set to Download context.SyncWay = SyncWay.Download; // JUST Before get changes, get the timestamp, to be sure to // get rows inserted / updated elsewhere since the sync is not over (context, serverTimestamp) = await this.RemoteProvider.GetLocalTimestampAsync(context, new MessageTimestamp { ScopeInfoTableName = this.LocalProvider.Configuration.ScopeInfoTableName, SerializationFormat = this.LocalProvider.Configuration.SerializationFormat }); if (cancellationToken.IsCancellationRequested) { cancellationToken.ThrowIfCancellationRequested(); } (context, serverBatchInfo, serverChangesSelected) = await this.RemoteProvider.GetChangeBatchAsync(context, new MessageGetChangesBatch { ScopeInfo = scope, Schema = this.LocalProvider.Configuration.Schema, Policy = serverPolicy, Filters = this.LocalProvider.Configuration.Filters, SerializationFormat = this.LocalProvider.Configuration.SerializationFormat }); if (cancellationToken.IsCancellationRequested) { cancellationToken.ThrowIfCancellationRequested(); } // Apply local changes // fromId : When applying rows, make sure it's identified as applied by this server scope fromId = serverScopeInfo.Id; // lastSyncTS : apply lines only if they are not modified since last client sync lastSyncTS = localScopeInfo.LastSyncTimestamp; // isNew : if IsNew, don't apply deleted rows from server isNew = localScopeInfo.IsNewScope; scope = new ScopeInfo { Id = fromId, IsNewScope = isNew, Timestamp = lastSyncTS }; (context, clientChangesApplied) = await this.LocalProvider.ApplyChangesAsync(context, new MessageApplyChanges { FromScope = scope, Schema = this.LocalProvider.Configuration.Schema, Policy = clientPolicy, ScopeInfoTableName = this.LocalProvider.Configuration.ScopeInfoTableName, Changes = serverBatchInfo, SerializationFormat = this.LocalProvider.Configuration.SerializationFormat }); context.TotalChangesDownloaded = clientChangesApplied.TotalAppliedChanges; context.TotalChangesUploaded = clientChangesSelected.TotalChangesSelected; context.TotalSyncErrors = clientChangesApplied.TotalAppliedChangesFailed; context.CompleteTime = DateTime.Now; serverScopeInfo.IsNewScope = false; localScopeReferenceInfo.IsNewScope = false; localScopeInfo.IsNewScope = false; serverScopeInfo.LastSync = context.CompleteTime; localScopeReferenceInfo.LastSync = context.CompleteTime; localScopeInfo.LastSync = context.CompleteTime; serverScopeInfo.LastSyncTimestamp = serverTimestamp; localScopeReferenceInfo.LastSyncTimestamp = serverTimestamp; localScopeInfo.LastSyncTimestamp = clientTimestamp; var duration = context.CompleteTime.Subtract(context.StartTime); serverScopeInfo.LastSyncDuration = duration.Ticks; localScopeReferenceInfo.LastSyncDuration = duration.Ticks; localScopeInfo.LastSyncDuration = duration.Ticks; serverScopeInfo.IsLocal = true; localScopeReferenceInfo.IsLocal = false; context = await this.RemoteProvider.WriteScopesAsync(context, new MessageWriteScopes { ScopeInfoTableName = this.LocalProvider.Configuration.ScopeInfoTableName, Scopes = new List <ScopeInfo> { serverScopeInfo, localScopeReferenceInfo }, SerializationFormat = this.LocalProvider.Configuration.SerializationFormat }); serverScopeInfo.IsLocal = false; localScopeInfo.IsLocal = true; if (cancellationToken.IsCancellationRequested) { cancellationToken.ThrowIfCancellationRequested(); } context = await this.LocalProvider.WriteScopesAsync(context, new MessageWriteScopes { ScopeInfoTableName = this.LocalProvider.Configuration.ScopeInfoTableName, Scopes = new List <ScopeInfo> { localScopeInfo, serverScopeInfo }, SerializationFormat = this.LocalProvider.Configuration.SerializationFormat }); if (cancellationToken.IsCancellationRequested) { cancellationToken.ThrowIfCancellationRequested(); } } catch (SyncException se) { Console.WriteLine($"Sync Exception: {se.Message}. Type:{se.Type}."); throw; } catch (Exception ex) { Console.WriteLine($"Unknwon Exception: {ex.Message}."); throw new SyncException(ex, SyncStage.None); } finally { // End the current session context = await this.RemoteProvider.EndSessionAsync(context); context = await this.LocalProvider.EndSessionAsync(context); this.SessionState = SyncSessionState.Ready; this.SessionStateChanged?.Invoke(this, this.SessionState); } return(context); }
/// <summary> /// Gets a batch of changes to synchronize when given batch size, /// destination knowledge, and change data retriever parameters. /// </summary> /// <returns>A DbSyncContext object that will be used to retrieve the modified data.</returns> //public virtual async Task<TimeSpan> PrepareArchiveAsync(string[] tables, int downloadBatchSizeInKB, string batchDirectory, ConflictResolutionPolicy policy, ICollection<FilterClause> filters) //{ // try // { // // We need to save // // the lasttimestamp when the zip generated for the client to be able to launch a sync since this ts // // IF the client is new and the SyncConfiguration object has the Archive property // var stopwatch = new Stopwatch(); // stopwatch.Start(); // SyncContext context; // ScopeInfo scopeInfo; // context = new SyncContext(Guid.NewGuid()) // { // SyncType = SyncType.Normal, // SyncWay = SyncWay.Download, // }; // scopeInfo = new ScopeInfo // { // IsNewScope = true // }; // // Read configuration // var config = await this.ReadSchemaAsync(tables); // // We want a batch zip // if (downloadBatchSizeInKB <= 0) // downloadBatchSizeInKB = 10000; // (var batchInfo, var changesSelected) = // await this.EnumerateChangesInBatchesInternal(context, scopeInfo, downloadBatchSizeInKB, config.Schema, batchDirectory, policy, filters); // var dir = batchInfo.GetDirectoryFullPath(); // var archiveFullName = string.Concat(batchDirectory, "\\", Path.GetRandomFileName()); // ZipFile.CreateFromDirectory(dir, archiveFullName, CompressionLevel.Fastest, false); // stopwatch.Stop(); // return stopwatch.Elapsed; // } // catch (Exception ex) // { // throw new SyncException(ex, SyncStage.TableChangesSelecting, this.ProviderTypeName); // } //} /// <summary> /// Generate an empty BatchInfo /// </summary> internal (BatchInfo, DatabaseChangesSelected) GetEmptyChanges(SyncContext context, ScopeInfo scopeInfo, int downloadBatchSizeInKB, string batchDirectory) { // Get config var isBatched = downloadBatchSizeInKB > 0; // create the in memory changes set var changesSet = new DmSet(SyncConfiguration.DMSET_NAME); // Create the batch info, in memory var batchInfo = new BatchInfo(!isBatched, batchDirectory); if (isBatched) { batchInfo.GenerateNewDirectoryName(); } // generate the batchpartinfo var bpi = batchInfo.GenerateBatchInfo(0, changesSet); bpi.IsLastBatch = true; // Create a new in-memory batch info with an the changes DmSet return(batchInfo, new DatabaseChangesSelected()); }