/// <summary> /// Generate an empty BatchInfo /// </summary> internal Task <(BatchInfo, DatabaseChangesSelected)> GetEmptyChangesAsync(MessageGetChangesBatch message) { // Get config var isBatched = message.BatchSize > 0; // Create the batch info, in memory var batchInfo = new BatchInfo(!isBatched, message.Schema, message.BatchDirectory);; this.Orchestrator.logger.LogDebug(SyncEventsId.GetChanges, new DatabaseChangesSelected()); // Create a new empty in-memory batch info return(Task.FromResult((batchInfo, new DatabaseChangesSelected()))); }
/// <summary> /// Generate an empty BatchInfo /// </summary> internal (BatchInfo, DatabaseChangesSelected) GetEmptyChanges(MessageGetChangesBatch message) { // Get config var isBatched = message.BatchSize > 0; // create the in memory changes set var changesSet = new SyncSet(message.Schema.ScopeName); // Create a Schema set without readonly tables, attached to memory changes foreach (var table in message.Schema.Tables) { DbSyncAdapter.CreateChangesTable(message.Schema.Tables[table.TableName, table.SchemaName], changesSet); } // Create the batch info, in memory var batchInfo = new BatchInfo(!isBatched, changesSet, message.BatchDirectory);; // add changes to batchInfo batchInfo.AddChanges(new SyncSet()); // Create a new empty in-memory batch info return(batchInfo, new DatabaseChangesSelected()); }
/// <summary> /// Gets a batch of changes to synchronize when given batch size, /// destination knowledge, and change data retriever parameters. /// </summary> /// <returns>A DbSyncContext object that will be used to retrieve the modified data.</returns> public virtual async Task <(SyncContext, BatchInfo, ChangesSelected)> GetChangeBatchAsync( SyncContext context, MessageGetChangesBatch message) { try { if (message.ScopeInfo == null) { throw new ArgumentNullException("scopeInfo", "Client scope info is null"); } // Check if the provider is not outdated var isOutdated = this.IsRemoteOutdated(); // Get a chance to make the sync even if it's outdated if (isOutdated) { var outdatedEventArgs = new OutdatedEventArgs(); this.SyncOutdated?.Invoke(this, outdatedEventArgs); if (outdatedEventArgs.Action != OutdatedSyncAction.Rollback) { context.SyncType = outdatedEventArgs.Action == OutdatedSyncAction.Reinitialize ? SyncType.Reinitialize : SyncType.ReinitializeWithUpload; } if (outdatedEventArgs.Action == OutdatedSyncAction.Rollback) { throw new OutOfDateException("The provider is out of date ! Try to make a Reinitialize sync"); } } // create local directory if (message.DownloadBatchSizeInKB > 0 && !String.IsNullOrEmpty(message.BatchDirectory) && !Directory.Exists(message.BatchDirectory)) { Directory.CreateDirectory(message.BatchDirectory); } // batch info containing changes BatchInfo batchInfo; // Statistics about changes that are selected ChangesSelected changesSelected; // if we try a Reinitialize action, don't get any changes from client // else get changes from batch or in memory methods if (context.SyncWay == SyncWay.Upload && context.SyncType == SyncType.Reinitialize) { (batchInfo, changesSelected) = this.GetEmptyChanges(context, message.ScopeInfo, message.DownloadBatchSizeInKB, message.BatchDirectory); } else if (message.DownloadBatchSizeInKB == 0) { (batchInfo, changesSelected) = await this.EnumerateChangesInternal(context, message.ScopeInfo, message.Schema, message.BatchDirectory, message.Policy, message.Filters); } else { (batchInfo, changesSelected) = await this.EnumerateChangesInBatchesInternal(context, message.ScopeInfo, message.DownloadBatchSizeInKB, message.Schema, message.BatchDirectory, message.Policy, message.Filters); } return(context, batchInfo, changesSelected); } catch (Exception ex) { throw new SyncException(ex, SyncStage.TableChangesSelecting, this.ProviderTypeName); } }
ApplyThenGetChangesAsync(ScopeInfo clientScope, BatchInfo clientBatchInfo, CancellationToken cancellationToken = default, IProgress <ProgressArgs> progress = null) { if (!this.StartTime.HasValue) { this.StartTime = DateTime.UtcNow; } // Get context or create a new one var ctx = this.GetContext(); long remoteClientTimestamp = 0L; DatabaseChangesSelected serverChangesSelected = null; DatabaseChangesApplied clientChangesApplied = null; BatchInfo serverBatchInfo = null; SyncSet schema = null; using var connection = this.Provider.CreateConnection(); try { ctx.SyncStage = SyncStage.ChangesApplying; //Direction set to Upload ctx.SyncWay = SyncWay.Upload; // Open connection await this.OpenConnectionAsync(connection, cancellationToken).ConfigureAwait(false); DbTransaction transaction; // Create two transactions // First one to commit changes // Second one to get changes now that everything is commited using (transaction = connection.BeginTransaction()) { await this.InterceptAsync(new TransactionOpenedArgs(ctx, connection, transaction), cancellationToken).ConfigureAwait(false); // Maybe here, get the schema from server, issue from client scope name // Maybe then compare the schema version from client scope with schema version issued from server // Maybe if different, raise an error ? // Get scope if exists // Getting server scope assumes we have already created the schema on server var scopeBuilder = this.GetScopeBuilder(this.Options.ScopeInfoTableName); var serverScopeInfo = await this.InternalGetScopeAsync <ServerScopeInfo>(ctx, DbScopeType.Server, clientScope.Name, scopeBuilder, connection, transaction, cancellationToken, progress).ConfigureAwait(false); // Should we ? if (serverScopeInfo.Schema == null) { throw new MissingRemoteOrchestratorSchemaException(); } // Deserialiaze schema schema = serverScopeInfo.Schema; // Create message containing everything we need to apply on server side var applyChanges = new MessageApplyChanges(Guid.Empty, clientScope.Id, false, clientScope.LastServerSyncTimestamp, schema, this.Setup, this.Options.ConflictResolutionPolicy, this.Options.DisableConstraintsOnApplyChanges, this.Options.UseBulkOperations, this.Options.CleanMetadatas, this.Options.CleanFolder, false, clientBatchInfo, this.Options.SerializerFactory); // Call provider to apply changes (ctx, clientChangesApplied) = await this.InternalApplyChangesAsync(ctx, applyChanges, connection, transaction, cancellationToken, progress).ConfigureAwait(false); await this.InterceptAsync(new TransactionCommitArgs(ctx, connection, transaction), cancellationToken).ConfigureAwait(false); // commit first transaction transaction.Commit(); } ctx.SyncStage = SyncStage.ChangesSelecting; ctx.ProgressPercentage = 0.55; using (transaction = connection.BeginTransaction()) { await this.InterceptAsync(new TransactionOpenedArgs(ctx, connection, transaction), cancellationToken).ConfigureAwait(false); //Direction set to Download ctx.SyncWay = SyncWay.Download; // JUST Before get changes, get the timestamp, to be sure to // get rows inserted / updated elsewhere since the sync is not over remoteClientTimestamp = await this.InternalGetLocalTimestampAsync(ctx, connection, transaction, cancellationToken, progress); // Get if we need to get all rows from the datasource var fromScratch = clientScope.IsNewScope || ctx.SyncType == SyncType.Reinitialize || ctx.SyncType == SyncType.ReinitializeWithUpload; var message = new MessageGetChangesBatch(clientScope.Id, Guid.Empty, fromScratch, clientScope.LastServerSyncTimestamp, schema, this.Setup, this.Options.BatchSize, this.Options.BatchDirectory, this.Options.SerializerFactory); // Call interceptor await this.InterceptAsync(new DatabaseChangesSelectingArgs(ctx, message, connection, transaction), cancellationToken).ConfigureAwait(false); // When we get the chnages from server, we create the batches if it's requested by the client // the batch decision comes from batchsize from client (ctx, serverBatchInfo, serverChangesSelected) = await this.InternalGetChangesAsync(ctx, message, connection, transaction, cancellationToken, progress).ConfigureAwait(false); if (cancellationToken.IsCancellationRequested) { cancellationToken.ThrowIfCancellationRequested(); } // generate the new scope item this.CompleteTime = DateTime.UtcNow; var scopeHistory = new ServerHistoryScopeInfo { Id = clientScope.Id, Name = clientScope.Name, LastSyncTimestamp = remoteClientTimestamp, LastSync = this.CompleteTime, LastSyncDuration = this.CompleteTime.Value.Subtract(this.StartTime.Value).Ticks, }; // Write scopes locally var scopeBuilder = this.GetScopeBuilder(this.Options.ScopeInfoTableName); await this.InternalSaveScopeAsync(ctx, DbScopeType.ServerHistory, scopeHistory, scopeBuilder, connection, transaction, cancellationToken, progress).ConfigureAwait(false); // Commit second transaction for getting changes await this.InterceptAsync(new TransactionCommitArgs(ctx, connection, transaction), cancellationToken).ConfigureAwait(false); transaction.Commit(); } // Event progress & interceptor await this.CloseConnectionAsync(connection, cancellationToken).ConfigureAwait(false); } catch (Exception ex) { RaiseError(ex); } finally { await this.CloseConnectionAsync(connection, cancellationToken).ConfigureAwait(false); } return(remoteClientTimestamp, serverBatchInfo, this.Options.ConflictResolutionPolicy, clientChangesApplied, serverChangesSelected); }
/// <summary> /// Generate an empty BatchInfo /// </summary> internal Task <(BatchInfo, DatabaseChangesSelected)> InternalGetEmptyChangesAsync(MessageGetChangesBatch message) { // Get config var isBatched = message.BatchSize > 0; // Create the batch info, in memory var batchInfo = new BatchInfo(!isBatched, message.Schema, message.BatchDirectory);; // Create a new empty in-memory batch info return(Task.FromResult((batchInfo, new DatabaseChangesSelected()))); }
/// <summary> /// Gets a batch of changes to synchronize when given batch size, /// destination knowledge, and change data retriever parameters. /// </summary> /// <returns>A DbSyncContext object that will be used to retrieve the modified data.</returns> internal virtual async Task <(SyncContext, BatchInfo, DatabaseChangesSelected)> InternalGetChangesAsync( SyncContext context, MessageGetChangesBatch message, DbConnection connection, DbTransaction transaction, CancellationToken cancellationToken, IProgress <ProgressArgs> progress) { // batch info containing changes BatchInfo batchInfo; // Statistics about changes that are selected DatabaseChangesSelected changesSelected; if (context.SyncWay == SyncWay.Upload && context.SyncType == SyncType.Reinitialize) { (batchInfo, changesSelected) = await this.InternalGetEmptyChangesAsync(message).ConfigureAwait(false); return(context, batchInfo, changesSelected); } // Call interceptor await this.InterceptAsync(new DatabaseChangesSelectingArgs(context, message, connection, transaction), cancellationToken).ConfigureAwait(false); // create local directory if (message.BatchSize > 0 && !string.IsNullOrEmpty(message.BatchDirectory) && !Directory.Exists(message.BatchDirectory)) { Directory.CreateDirectory(message.BatchDirectory); } changesSelected = new DatabaseChangesSelected(); // numbers of batch files generated var batchIndex = 0; // Check if we are in batch mode var isBatch = message.BatchSize > 0; // Create a batch info in memory (if !isBatch) or serialized on disk (if isBatch) // batchinfo generate a schema clone with scope columns if needed batchInfo = new BatchInfo(!isBatch, message.Schema, message.BatchDirectory); // Clean SyncSet, we will add only tables we need in the batch info var changesSet = new SyncSet(); var cptSyncTable = 0; var currentProgress = context.ProgressPercentage; foreach (var syncTable in message.Schema.Tables) { // tmp count of table for report progress pct cptSyncTable++; // Only table schema is replicated, no datas are applied if (syncTable.SyncDirection == SyncDirection.None) { continue; } // if we are in upload stage, so check if table is not download only if (context.SyncWay == SyncWay.Upload && syncTable.SyncDirection == SyncDirection.DownloadOnly) { continue; } // if we are in download stage, so check if table is not download only if (context.SyncWay == SyncWay.Download && syncTable.SyncDirection == SyncDirection.UploadOnly) { continue; } // Get Command var selectIncrementalChangesCommand = await this.GetSelectChangesCommandAsync(context, syncTable, message.Setup, message.IsNew, connection, transaction); // Set parameters this.SetSelectChangesCommonParameters(context, syncTable, message.ExcludingScopeId, message.IsNew, message.LastTimestamp, selectIncrementalChangesCommand); // launch interceptor if any var args = new TableChangesSelectingArgs(context, syncTable, selectIncrementalChangesCommand, connection, transaction); await this.InterceptAsync(args, cancellationToken).ConfigureAwait(false); if (!args.Cancel && args.Command != null) { // Statistics var tableChangesSelected = new TableChangesSelected(syncTable.TableName, syncTable.SchemaName); // Create a chnages table with scope columns var changesSetTable = DbSyncAdapter.CreateChangesTable(message.Schema.Tables[syncTable.TableName, syncTable.SchemaName], changesSet); // Get the reader using var dataReader = await args.Command.ExecuteReaderAsync().ConfigureAwait(false); // memory size total double rowsMemorySize = 0L; while (dataReader.Read()) { // Create a row from dataReader var row = CreateSyncRowFromReader(dataReader, changesSetTable); // Add the row to the changes set changesSetTable.Rows.Add(row); // Set the correct state to be applied if (row.RowState == DataRowState.Deleted) { tableChangesSelected.Deletes++; } else if (row.RowState == DataRowState.Modified) { tableChangesSelected.Upserts++; } // calculate row size if in batch mode if (isBatch) { var fieldsSize = ContainerTable.GetRowSizeFromDataRow(row.ToArray()); var finalFieldSize = fieldsSize / 1024d; if (finalFieldSize > message.BatchSize) { throw new RowOverSizedException(finalFieldSize.ToString()); } // Calculate the new memory size rowsMemorySize += finalFieldSize; // Next line if we don't reach the batch size yet. if (rowsMemorySize <= message.BatchSize) { continue; } // Check interceptor var batchTableChangesSelectedArgs = new TableChangesSelectedArgs(context, changesSetTable, tableChangesSelected, connection, transaction); await this.InterceptAsync(batchTableChangesSelectedArgs, cancellationToken).ConfigureAwait(false); // add changes to batchinfo await batchInfo.AddChangesAsync(changesSet, batchIndex, false, message.SerializerFactory, this).ConfigureAwait(false); // increment batch index batchIndex++; // we know the datas are serialized here, so we can flush the set changesSet.Clear(); // Recreate an empty ContainerSet and a ContainerTable changesSet = new SyncSet(); changesSetTable = DbSyncAdapter.CreateChangesTable(message.Schema.Tables[syncTable.TableName, syncTable.SchemaName], changesSet); // Init the row memory size rowsMemorySize = 0L; } } dataReader.Close(); // We don't report progress if no table changes is empty, to limit verbosity if (tableChangesSelected.Deletes > 0 || tableChangesSelected.Upserts > 0) { changesSelected.TableChangesSelected.Add(tableChangesSelected); } // even if no rows raise the interceptor var tableChangesSelectedArgs = new TableChangesSelectedArgs(context, changesSetTable, tableChangesSelected, connection, transaction); await this.InterceptAsync(tableChangesSelectedArgs, cancellationToken).ConfigureAwait(false); context.ProgressPercentage = currentProgress + (cptSyncTable * 0.2d / message.Schema.Tables.Count); // only raise report progress if we have something if (tableChangesSelectedArgs.TableChangesSelected.TotalChanges > 0) { this.ReportProgress(context, progress, tableChangesSelectedArgs); } } } // We are in batch mode, and we are at the last batchpart info // Even if we don't have rows inside, we return the changesSet, since it contains at least schema if (changesSet != null && changesSet.HasTables && changesSet.HasRows) { await batchInfo.AddChangesAsync(changesSet, batchIndex, true, message.SerializerFactory, this).ConfigureAwait(false); } //Set the total rows count contained in the batch info batchInfo.RowsCount = changesSelected.TotalChangesSelected; // Check the last index as the last batch batchInfo.EnsureLastBatch(); // Raise database changes selected if (changesSelected.TotalChangesSelected > 0 || changesSelected.TotalChangesSelectedDeletes > 0 || changesSelected.TotalChangesSelectedUpdates > 0) { var databaseChangesSelectedArgs = new DatabaseChangesSelectedArgs(context, message.LastTimestamp, batchInfo, changesSelected, connection); this.ReportProgress(context, progress, databaseChangesSelectedArgs); await this.InterceptAsync(databaseChangesSelectedArgs, cancellationToken).ConfigureAwait(false); } return(context, batchInfo, changesSelected); }
/// <summary> /// Gets changes rows count estimation, /// </summary> internal virtual async Task <(SyncContext, DatabaseChangesSelected)> InternalGetEstimatedChangesCountAsync( SyncContext context, MessageGetChangesBatch message, DbConnection connection, DbTransaction transaction, CancellationToken cancellationToken, IProgress <ProgressArgs> progress) { // Call interceptor await this.InterceptAsync(new DatabaseChangesSelectingArgs(context, message, connection, transaction), cancellationToken).ConfigureAwait(false); // Create stats object to store changes count var changes = new DatabaseChangesSelected(); if (context.SyncWay == SyncWay.Upload && context.SyncType == SyncType.Reinitialize) { return(context, changes); } foreach (var syncTable in message.Schema.Tables) { // Only table schema is replicated, no datas are applied if (syncTable.SyncDirection == SyncDirection.None) { continue; } // if we are in upload stage, so check if table is not download only if (context.SyncWay == SyncWay.Upload && syncTable.SyncDirection == SyncDirection.DownloadOnly) { continue; } // if we are in download stage, so check if table is not download only if (context.SyncWay == SyncWay.Download && syncTable.SyncDirection == SyncDirection.UploadOnly) { continue; } // Get Command var command = await this.GetSelectChangesCommandAsync(context, syncTable, message.Setup, message.IsNew, connection, transaction); // Set parameters this.SetSelectChangesCommonParameters(context, syncTable, message.ExcludingScopeId, message.IsNew, message.LastTimestamp, command); // launch interceptor if any var args = new TableChangesSelectingArgs(context, syncTable, command, connection, transaction); await this.InterceptAsync(args, cancellationToken).ConfigureAwait(false); if (args.Cancel || args.Command == null) { continue; } // Statistics var tableChangesSelected = new TableChangesSelected(syncTable.TableName, syncTable.SchemaName); // Get the reader using var dataReader = await args.Command.ExecuteReaderAsync().ConfigureAwait(false); while (dataReader.Read()) { bool isTombstone = false; for (var i = 0; i < dataReader.FieldCount; i++) { if (dataReader.GetName(i) == "sync_row_is_tombstone") { isTombstone = Convert.ToInt64(dataReader.GetValue(i)) > 0; break; } } // Set the correct state to be applied if (isTombstone) { tableChangesSelected.Deletes++; } else { tableChangesSelected.Upserts++; } } dataReader.Close(); // Check interceptor var changesArgs = new TableChangesSelectedArgs(context, null, tableChangesSelected, connection, transaction); await this.InterceptAsync(changesArgs, cancellationToken).ConfigureAwait(false); if (tableChangesSelected.Deletes > 0 || tableChangesSelected.Upserts > 0) { changes.TableChangesSelected.Add(tableChangesSelected); } } // Raise database changes selected var databaseChangesSelectedArgs = new DatabaseChangesSelectedArgs(context, message.LastTimestamp, null, changes, connection); this.ReportProgress(context, progress, databaseChangesSelectedArgs); await this.InterceptAsync(databaseChangesSelectedArgs, cancellationToken).ConfigureAwait(false); return(context, changes); }
GetEstimatedChangesCountAsync(ScopeInfo localScopeInfo = null, CancellationToken cancellationToken = default, IProgress <ProgressArgs> progress = null) { if (!this.StartTime.HasValue) { this.StartTime = DateTime.UtcNow; } // Output long clientTimestamp = 0L; DatabaseChangesSelected clientChangesSelected = null; // Get context or create a new one var ctx = this.GetContext(); using (var connection = this.Provider.CreateConnection()) { try { ctx.SyncStage = SyncStage.ChangesSelecting; // Open connection await this.OpenConnectionAsync(connection, cancellationToken).ConfigureAwait(false); using (var transaction = connection.BeginTransaction()) { await this.InterceptAsync(new TransactionOpenedArgs(ctx, connection, transaction), cancellationToken).ConfigureAwait(false); // Get local scope, if not provided if (localScopeInfo == null) { ctx = await this.Provider.EnsureClientScopeAsync(ctx, this.Options.ScopeInfoTableName, connection, transaction, cancellationToken, progress).ConfigureAwait(false); (ctx, localScopeInfo) = await this.Provider.GetClientScopeAsync(ctx, this.Options.ScopeInfoTableName, this.ScopeName, connection, transaction, cancellationToken, progress).ConfigureAwait(false); } // If no schema in the client scope. Maybe the client scope table does not exists, or we never get the schema from server if (localScopeInfo.Schema == null) { throw new MissingLocalOrchestratorSchemaException(); } this.logger.LogInformation(SyncEventsId.GetClientScope, localScopeInfo); // On local, we don't want to chase rows from "others" // We just want our local rows, so we dont exclude any remote scope id, by setting scope id to NULL Guid?remoteScopeId = null; // lastSyncTS : get lines inserted / updated / deteleted after the last sync commited var lastSyncTS = localScopeInfo.LastSyncTimestamp; // isNew : If isNew, lasttimestamp is not correct, so grab all var isNew = localScopeInfo.IsNewScope; //Direction set to Upload ctx.SyncWay = SyncWay.Upload; // JUST before the whole process, get the timestamp, to be sure to // get rows inserted / updated elsewhere since the sync is not over clientTimestamp = await this.Provider.GetLocalTimestampAsync(ctx, connection, transaction, cancellationToken, progress); // Creating the message // Since it's an estimated count, we don't need to create batches, so we hard code the batchsize to 0 var message = new MessageGetChangesBatch(remoteScopeId, localScopeInfo.Id, isNew, lastSyncTS, localScopeInfo.Schema, this.Setup, 0, this.Options.BatchDirectory); // Call interceptor await this.InterceptAsync(new DatabaseChangesSelectingArgs(ctx, message, connection, transaction), cancellationToken).ConfigureAwait(false); this.logger.LogDebug(SyncEventsId.GetChanges, message); // Locally, if we are new, no need to get changes if (isNew) { clientChangesSelected = new DatabaseChangesSelected(); } else { (ctx, clientChangesSelected) = await this.Provider.GetEstimatedChangesCountAsync(ctx, message, connection, transaction, cancellationToken, progress).ConfigureAwait(false); } await this.InterceptAsync(new TransactionCommitArgs(ctx, connection, transaction), cancellationToken).ConfigureAwait(false); transaction.Commit(); } // Event progress & interceptor ctx.SyncStage = SyncStage.ChangesSelected; await this.CloseConnectionAsync(connection, cancellationToken).ConfigureAwait(false); } catch (Exception ex) { RaiseError(ex); } finally { if (connection != null && connection.State != ConnectionState.Closed) { connection.Close(); } } this.logger.LogInformation(SyncEventsId.GetChanges, new { ClientTimestamp = clientTimestamp, ClientChangesSelected = clientChangesSelected }); return(clientTimestamp, clientChangesSelected); } }
/// <summary> /// Gets changes rows count estimation, /// </summary> public virtual async Task <(SyncContext, DatabaseChangesSelected)> GetEstimatedChangesCountAsync( SyncContext context, MessageGetChangesBatch message, DbConnection connection, DbTransaction transaction, CancellationToken cancellationToken, IProgress <ProgressArgs> progress) { this.Orchestrator.logger.LogDebug(SyncEventsId.GetChanges, message); // Create stats object to store changes count var changes = new DatabaseChangesSelected(); if (context.SyncWay == SyncWay.Upload && context.SyncType == SyncType.Reinitialize) { return(context, changes); } foreach (var syncTable in message.Schema.Tables) { this.Orchestrator.logger.LogDebug(SyncEventsId.GetChanges, syncTable); // if we are in upload stage, so check if table is not download only if (context.SyncWay == SyncWay.Upload && syncTable.SyncDirection == SyncDirection.DownloadOnly) { continue; } // if we are in download stage, so check if table is not download only if (context.SyncWay == SyncWay.Download && syncTable.SyncDirection == SyncDirection.UploadOnly) { continue; } var tableBuilder = this.GetTableBuilder(syncTable, message.Setup); var syncAdapter = tableBuilder.CreateSyncAdapter(); // launch interceptor if any await this.Orchestrator.InterceptAsync(new TableChangesSelectingArgs(context, syncTable, connection, transaction), cancellationToken).ConfigureAwait(false); // Get Command var selectIncrementalChangesCommand = await this.GetSelectChangesCommandAsync(context, syncAdapter, syncTable, message.IsNew, connection, transaction); // Set parameters this.SetSelectChangesCommonParameters(context, syncTable, message.ExcludingScopeId, message.IsNew, message.LastTimestamp, selectIncrementalChangesCommand); // log this.Orchestrator.logger.LogDebug(SyncEventsId.GetChanges, selectIncrementalChangesCommand); // Statistics var tableChangesSelected = new TableChangesSelected(syncTable.TableName, syncTable.SchemaName); // Get the reader using (var dataReader = await selectIncrementalChangesCommand.ExecuteReaderAsync().ConfigureAwait(false)) { while (dataReader.Read()) { bool isTombstone = false; for (var i = 0; i < dataReader.FieldCount; i++) { if (dataReader.GetName(i) == "sync_row_is_tombstone") { isTombstone = Convert.ToInt64(dataReader.GetValue(i)) > 0; break; } } // Set the correct state to be applied if (isTombstone) { tableChangesSelected.Deletes++; } else { tableChangesSelected.Upserts++; } } } if (tableChangesSelected.Deletes > 0 || tableChangesSelected.Upserts > 0) { changes.TableChangesSelected.Add(tableChangesSelected); } } return(context, changes); }
public DatabaseChangesSelectingArgs(SyncContext context, MessageGetChangesBatch changesRequest, DbConnection connection, DbTransaction transaction) : base(context, connection, transaction) { this.ChangesRequest = changesRequest; }
/// <summary> /// Gets a batch of changes to synchronize when given batch size, /// destination knowledge, and change data retriever parameters. /// </summary> /// <returns>A DbSyncContext object that will be used to retrieve the modified data.</returns> public virtual async Task <(SyncContext, BatchInfo, DatabaseChangesSelected)> GetChangeBatchAsync( SyncContext context, MessageGetChangesBatch message, DbConnection connection, DbTransaction transaction, CancellationToken cancellationToken, IProgress <ProgressArgs> progress = null) { // batch info containing changes BatchInfo batchInfo; // Statistics about changes that are selected DatabaseChangesSelected changesSelected; if (context.SyncWay == SyncWay.Upload && context.SyncType == SyncType.Reinitialize) { (batchInfo, changesSelected) = this.GetEmptyChanges(message); return(context, batchInfo, changesSelected); } // Check if the provider is not outdated var isOutdated = this.IsRemoteOutdated(); // Get a chance to make the sync even if it's outdated if (isOutdated) { var outdatedArgs = new OutdatedArgs(context, null, null); // Interceptor await this.InterceptAsync(outdatedArgs).ConfigureAwait(false); if (outdatedArgs.Action != OutdatedAction.Rollback) { context.SyncType = outdatedArgs.Action == OutdatedAction.Reinitialize ? SyncType.Reinitialize : SyncType.ReinitializeWithUpload; } if (outdatedArgs.Action == OutdatedAction.Rollback) { throw new OutOfDateException(); } } // create local directory if (message.BatchSize > 0 && !string.IsNullOrEmpty(message.BatchDirectory) && !Directory.Exists(message.BatchDirectory)) { Directory.CreateDirectory(message.BatchDirectory); } // numbers of batch files generated var batchIndex = 0; // Check if we are in batch mode var isBatch = message.BatchSize > 0; // Create stats object to store changes count var changes = new DatabaseChangesSelected(); // create the in memory changes set var changesSet = new SyncSet(message.Schema.ScopeName); // Create a Schema set without readonly columns, attached to memory changes foreach (var table in message.Schema.Tables) { DbSyncAdapter.CreateChangesTable(message.Schema.Tables[table.TableName, table.SchemaName], changesSet); } // Create a batch info in memory (if !isBatch) or serialized on disk (if isBatch) // batchinfo generate a schema clone with scope columns if needed batchInfo = new BatchInfo(!isBatch, changesSet, message.BatchDirectory); // Clear tables, we will add only the ones we need in the batch info changesSet.Clear(); foreach (var syncTable in message.Schema.Tables) { // if we are in upload stage, so check if table is not download only if (context.SyncWay == SyncWay.Upload && syncTable.SyncDirection == SyncDirection.DownloadOnly) { continue; } // if we are in download stage, so check if table is not download only if (context.SyncWay == SyncWay.Download && syncTable.SyncDirection == SyncDirection.UploadOnly) { continue; } var tableBuilder = this.GetTableBuilder(syncTable); var syncAdapter = tableBuilder.CreateSyncAdapter(connection, transaction); // raise before event context.SyncStage = SyncStage.TableChangesSelecting; var tableChangesSelectingArgs = new TableChangesSelectingArgs(context, syncTable.TableName, connection, transaction); // launch interceptor if any await this.InterceptAsync(tableChangesSelectingArgs).ConfigureAwait(false); // Get Command var selectIncrementalChangesCommand = this.GetSelectChangesCommand(context, syncAdapter, syncTable, message.IsNew); // Set parameters this.SetSelectChangesCommonParameters(context, syncTable, message.ExcludingScopeId, message.IsNew, message.LastTimestamp, selectIncrementalChangesCommand); // Statistics var tableChangesSelected = new TableChangesSelected(syncTable.TableName); // Get the reader using (var dataReader = selectIncrementalChangesCommand.ExecuteReader()) { // memory size total double rowsMemorySize = 0L; // Create a chnages table with scope columns var changesSetTable = DbSyncAdapter.CreateChangesTable(message.Schema.Tables[syncTable.TableName, syncTable.SchemaName], changesSet); while (dataReader.Read()) { // Create a row from dataReader var row = CreateSyncRowFromReader(dataReader, changesSetTable); // Add the row to the changes set changesSetTable.Rows.Add(row); // Set the correct state to be applied if (row.RowState == DataRowState.Deleted) { tableChangesSelected.Deletes++; } else if (row.RowState == DataRowState.Modified) { tableChangesSelected.Upserts++; } // calculate row size if in batch mode if (isBatch) { var fieldsSize = ContainerTable.GetRowSizeFromDataRow(row.ToArray()); var finalFieldSize = fieldsSize / 1024d; if (finalFieldSize > message.BatchSize) { throw new RowOverSizedException(finalFieldSize.ToString()); } // Calculate the new memory size rowsMemorySize += finalFieldSize; // Next line if we don't reach the batch size yet. if (rowsMemorySize <= message.BatchSize) { continue; } // add changes to batchinfo batchInfo.AddChanges(changesSet, batchIndex, false); // increment batch index batchIndex++; // we know the datas are serialized here, so we can flush the set changesSet.Clear(); // Recreate an empty ContainerSet and a ContainerTable changesSet = new SyncSet(message.Schema.ScopeName); changesSetTable = DbSyncAdapter.CreateChangesTable(message.Schema.Tables[syncTable.TableName, syncTable.SchemaName], changesSet); // Init the row memory size rowsMemorySize = 0L; } } } selectIncrementalChangesCommand.Dispose(); context.SyncStage = SyncStage.TableChangesSelected; if (tableChangesSelected.Deletes > 0 || tableChangesSelected.Upserts > 0) { changes.TableChangesSelected.Add(tableChangesSelected); } // Event progress & interceptor context.SyncStage = SyncStage.TableChangesSelected; var tableChangesSelectedArgs = new TableChangesSelectedArgs(context, tableChangesSelected, connection, transaction); this.ReportProgress(context, progress, tableChangesSelectedArgs); await this.InterceptAsync(tableChangesSelectedArgs).ConfigureAwait(false); } // We are in batch mode, and we are at the last batchpart info // Even if we don't have rows inside, we return the changesSet, since it contains at leaset schema if (changesSet != null && changesSet.HasTables) { batchInfo.AddChanges(changesSet, batchIndex, true); } // Check the last index as the last batch batchInfo.EnsureLastBatch(); return(context, batchInfo, changes); }