/// <summary> /// Generate an empty BatchInfo /// </summary> internal async Task <(BatchInfo, DatabaseChangesSelected)> GetEmptyChangesAsync(MessageGetChangesBatch message) { // Get config var isBatched = message.BatchSize > 0; // create the in memory changes set var changesSet = new SyncSet(message.Schema.ScopeName); // Create a Schema set without readonly tables, attached to memory changes foreach (var table in message.Schema.Tables) { DbSyncAdapter.CreateChangesTable(message.Schema.Tables[table.TableName, table.SchemaName], changesSet); } // Create the batch info, in memory var batchInfo = new BatchInfo(!isBatched, changesSet, message.BatchDirectory);; // add changes to batchInfo await batchInfo.AddChangesAsync(new SyncSet()).ConfigureAwait(false); // Create a new empty in-memory batch info return(batchInfo, new DatabaseChangesSelected()); }
/// <summary> /// Gets a batch of changes to synchronize when given batch size, /// destination knowledge, and change data retriever parameters. /// </summary> /// <returns>A DbSyncContext object that will be used to retrieve the modified data.</returns> internal virtual async Task <(SyncContext, BatchInfo, DatabaseChangesSelected)> InternalGetChangesAsync( SyncContext context, MessageGetChangesBatch message, DbConnection connection, DbTransaction transaction, CancellationToken cancellationToken, IProgress <ProgressArgs> progress) { // batch info containing changes BatchInfo batchInfo; // Statistics about changes that are selected DatabaseChangesSelected changesSelected; if (context.SyncWay == SyncWay.Upload && context.SyncType == SyncType.Reinitialize) { (batchInfo, changesSelected) = await this.InternalGetEmptyChangesAsync(message).ConfigureAwait(false); return(context, batchInfo, changesSelected); } // Call interceptor await this.InterceptAsync(new DatabaseChangesSelectingArgs(context, message, connection, transaction), cancellationToken).ConfigureAwait(false); // create local directory if (message.BatchSize > 0 && !string.IsNullOrEmpty(message.BatchDirectory) && !Directory.Exists(message.BatchDirectory)) { Directory.CreateDirectory(message.BatchDirectory); } changesSelected = new DatabaseChangesSelected(); // numbers of batch files generated var batchIndex = 0; // Check if we are in batch mode var isBatch = message.BatchSize > 0; // Create a batch info in memory (if !isBatch) or serialized on disk (if isBatch) // batchinfo generate a schema clone with scope columns if needed batchInfo = new BatchInfo(!isBatch, message.Schema, message.BatchDirectory); // Clean SyncSet, we will add only tables we need in the batch info var changesSet = new SyncSet(); var cptSyncTable = 0; var currentProgress = context.ProgressPercentage; foreach (var syncTable in message.Schema.Tables) { // tmp count of table for report progress pct cptSyncTable++; // Only table schema is replicated, no datas are applied if (syncTable.SyncDirection == SyncDirection.None) { continue; } // if we are in upload stage, so check if table is not download only if (context.SyncWay == SyncWay.Upload && syncTable.SyncDirection == SyncDirection.DownloadOnly) { continue; } // if we are in download stage, so check if table is not download only if (context.SyncWay == SyncWay.Download && syncTable.SyncDirection == SyncDirection.UploadOnly) { continue; } // Get Command var selectIncrementalChangesCommand = await this.GetSelectChangesCommandAsync(context, syncTable, message.Setup, message.IsNew, connection, transaction); // Set parameters this.SetSelectChangesCommonParameters(context, syncTable, message.ExcludingScopeId, message.IsNew, message.LastTimestamp, selectIncrementalChangesCommand); // launch interceptor if any var args = new TableChangesSelectingArgs(context, syncTable, selectIncrementalChangesCommand, connection, transaction); await this.InterceptAsync(args, cancellationToken).ConfigureAwait(false); if (!args.Cancel && args.Command != null) { // Statistics var tableChangesSelected = new TableChangesSelected(syncTable.TableName, syncTable.SchemaName); // Create a chnages table with scope columns var changesSetTable = DbSyncAdapter.CreateChangesTable(message.Schema.Tables[syncTable.TableName, syncTable.SchemaName], changesSet); // Get the reader using var dataReader = await args.Command.ExecuteReaderAsync().ConfigureAwait(false); // memory size total double rowsMemorySize = 0L; while (dataReader.Read()) { // Create a row from dataReader var row = CreateSyncRowFromReader(dataReader, changesSetTable); // Add the row to the changes set changesSetTable.Rows.Add(row); // Set the correct state to be applied if (row.RowState == DataRowState.Deleted) { tableChangesSelected.Deletes++; } else if (row.RowState == DataRowState.Modified) { tableChangesSelected.Upserts++; } // calculate row size if in batch mode if (isBatch) { var fieldsSize = ContainerTable.GetRowSizeFromDataRow(row.ToArray()); var finalFieldSize = fieldsSize / 1024d; if (finalFieldSize > message.BatchSize) { throw new RowOverSizedException(finalFieldSize.ToString()); } // Calculate the new memory size rowsMemorySize += finalFieldSize; // Next line if we don't reach the batch size yet. if (rowsMemorySize <= message.BatchSize) { continue; } // Check interceptor var batchTableChangesSelectedArgs = new TableChangesSelectedArgs(context, changesSetTable, tableChangesSelected, connection, transaction); await this.InterceptAsync(batchTableChangesSelectedArgs, cancellationToken).ConfigureAwait(false); // add changes to batchinfo await batchInfo.AddChangesAsync(changesSet, batchIndex, false, message.SerializerFactory, this).ConfigureAwait(false); // increment batch index batchIndex++; // we know the datas are serialized here, so we can flush the set changesSet.Clear(); // Recreate an empty ContainerSet and a ContainerTable changesSet = new SyncSet(); changesSetTable = DbSyncAdapter.CreateChangesTable(message.Schema.Tables[syncTable.TableName, syncTable.SchemaName], changesSet); // Init the row memory size rowsMemorySize = 0L; } } dataReader.Close(); // We don't report progress if no table changes is empty, to limit verbosity if (tableChangesSelected.Deletes > 0 || tableChangesSelected.Upserts > 0) { changesSelected.TableChangesSelected.Add(tableChangesSelected); } // even if no rows raise the interceptor var tableChangesSelectedArgs = new TableChangesSelectedArgs(context, changesSetTable, tableChangesSelected, connection, transaction); await this.InterceptAsync(tableChangesSelectedArgs, cancellationToken).ConfigureAwait(false); context.ProgressPercentage = currentProgress + (cptSyncTable * 0.2d / message.Schema.Tables.Count); // only raise report progress if we have something if (tableChangesSelectedArgs.TableChangesSelected.TotalChanges > 0) { this.ReportProgress(context, progress, tableChangesSelectedArgs); } } } // We are in batch mode, and we are at the last batchpart info // Even if we don't have rows inside, we return the changesSet, since it contains at least schema if (changesSet != null && changesSet.HasTables && changesSet.HasRows) { await batchInfo.AddChangesAsync(changesSet, batchIndex, true, message.SerializerFactory, this).ConfigureAwait(false); } //Set the total rows count contained in the batch info batchInfo.RowsCount = changesSelected.TotalChangesSelected; // Check the last index as the last batch batchInfo.EnsureLastBatch(); // Raise database changes selected if (changesSelected.TotalChangesSelected > 0 || changesSelected.TotalChangesSelectedDeletes > 0 || changesSelected.TotalChangesSelectedUpdates > 0) { var databaseChangesSelectedArgs = new DatabaseChangesSelectedArgs(context, message.LastTimestamp, batchInfo, changesSelected, connection); this.ReportProgress(context, progress, databaseChangesSelectedArgs); await this.InterceptAsync(databaseChangesSelectedArgs, cancellationToken).ConfigureAwait(false); } return(context, batchInfo, changesSelected); }
/// <summary> /// update configuration object with tables desc from server database /// </summary> public virtual async Task <(SyncContext, BatchInfo)> CreateSnapshotAsync(SyncContext context, SyncSet schema, SyncSetup setup, DbConnection connection, DbTransaction transaction, string snapshotDirectory, int batchSize, long remoteClientTimestamp, CancellationToken cancellationToken, IProgress <ProgressArgs> progress = null) { // create local directory if (!Directory.Exists(snapshotDirectory)) { this.Orchestrator.logger.LogDebug(SyncEventsId.CreateDirectory, new { SnapshotDirectory = snapshotDirectory }); Directory.CreateDirectory(snapshotDirectory); } // cleansing scope name var directoryScopeName = new string(context.ScopeName.Where(char.IsLetterOrDigit).ToArray()); var directoryFullPath = Path.Combine(snapshotDirectory, directoryScopeName); // create local directory with scope inside if (!Directory.Exists(directoryFullPath)) { this.Orchestrator.logger.LogDebug(SyncEventsId.CreateDirectory, new { DirectoryFullPath = directoryFullPath }); Directory.CreateDirectory(directoryFullPath); } // numbers of batch files generated var batchIndex = 0; // create the in memory changes set var changesSet = new SyncSet(); var sb = new StringBuilder(); var underscore = ""; if (context.Parameters != null) { foreach (var p in context.Parameters.OrderBy(p => p.Name)) { var cleanValue = new string(p.Value.ToString().Where(char.IsLetterOrDigit).ToArray()); var cleanName = new string(p.Name.Where(char.IsLetterOrDigit).ToArray()); sb.Append($"{underscore}{cleanName}_{cleanValue}"); underscore = "_"; } } var directoryName = sb.ToString(); directoryName = string.IsNullOrEmpty(directoryName) ? "ALL" : directoryName; // batchinfo generate a schema clone with scope columns if needed var batchInfo = new BatchInfo(false, schema, directoryFullPath, directoryName); // Delete directory if already exists directoryFullPath = Path.Combine(directoryFullPath, directoryName); if (Directory.Exists(directoryFullPath)) { this.Orchestrator.logger.LogDebug(SyncEventsId.DropDirectory, new { DirectoryFullPath = directoryFullPath }); Directory.Delete(directoryFullPath, true); } foreach (var syncTable in schema.Tables) { var tableBuilder = this.GetTableBuilder(syncTable, setup); var syncAdapter = tableBuilder.CreateSyncAdapter(); // launch interceptor if any await this.Orchestrator.InterceptAsync(new TableChangesSelectingArgs(context, syncTable, connection, transaction), cancellationToken).ConfigureAwait(false); // Get Select initialize changes command var selectIncrementalChangesCommand = await this.GetSelectChangesCommandAsync(context, syncAdapter, syncTable, true, connection, transaction); // Set parameters this.SetSelectChangesCommonParameters(context, syncTable, null, true, 0, selectIncrementalChangesCommand); // log this.Orchestrator.logger.LogDebug(SyncEventsId.CreateSnapshot, new { SelectChangesCommandText = selectIncrementalChangesCommand.CommandText, ExcludingScopeId = Guid.Empty, IsNew = true, LastTimestamp = 0 }); // Get the reader using (var dataReader = await selectIncrementalChangesCommand.ExecuteReaderAsync().ConfigureAwait(false)) { // memory size total double rowsMemorySize = 0L; // Create a chnages table with scope columns var changesSetTable = SyncAdapter.CreateChangesTable(schema.Tables[syncTable.TableName, syncTable.SchemaName], changesSet); while (dataReader.Read()) { // Create a row from dataReader var row = CreateSyncRowFromReader(dataReader, changesSetTable); // Add the row to the changes set changesSetTable.Rows.Add(row); // Log trace row this.Orchestrator.logger.LogTrace(SyncEventsId.CreateSnapshot, row); var fieldsSize = ContainerTable.GetRowSizeFromDataRow(row.ToArray()); var finalFieldSize = fieldsSize / 1024d; if (finalFieldSize > batchSize) { throw new RowOverSizedException(finalFieldSize.ToString()); } // Calculate the new memory size rowsMemorySize += finalFieldSize; // Next line if we don't reach the batch size yet. if (rowsMemorySize <= batchSize) { continue; } // add changes to batchinfo await batchInfo.AddChangesAsync(changesSet, batchIndex, false, this.Orchestrator).ConfigureAwait(false); this.Orchestrator.logger.LogDebug(SyncEventsId.CreateBatch, changesSet); // increment batch index batchIndex++; // we know the datas are serialized here, so we can flush the set changesSet.Clear(); // Recreate an empty ContainerSet and a ContainerTable changesSet = new SyncSet(); changesSetTable = SyncAdapter.CreateChangesTable(schema.Tables[syncTable.TableName, syncTable.SchemaName], changesSet); // Init the row memory size rowsMemorySize = 0L; } } //selectIncrementalChangesCommand.Dispose(); } if (changesSet != null && changesSet.HasTables) { await batchInfo.AddChangesAsync(changesSet, batchIndex, true, this.Orchestrator).ConfigureAwait(false); this.Orchestrator.logger.LogDebug(SyncEventsId.CreateBatch, changesSet); } // Check the last index as the last batch batchInfo.EnsureLastBatch(); batchInfo.Timestamp = remoteClientTimestamp; // Serialize on disk. var jsonConverter = new JsonConverter <BatchInfo>(); var summaryFileName = Path.Combine(directoryFullPath, "summary.json"); using (var f = new FileStream(summaryFileName, FileMode.CreateNew, FileAccess.ReadWrite)) { this.Orchestrator.logger.LogDebug(SyncEventsId.CreateSnapshotSummary, batchInfo); var bytes = await jsonConverter.SerializeAsync(batchInfo).ConfigureAwait(false); f.Write(bytes, 0, bytes.Length); } return(context, batchInfo); }
GetChangesAsync(ScopeInfo clientScope, CancellationToken cancellationToken = default, IProgress <ProgressArgs> progress = null) { SyncSet schema; // Get context or create a new one var ctx = this.GetContext(); if (!this.StartTime.HasValue) { this.StartTime = DateTime.UtcNow; } ServerScopeInfo serverScopeInfo; // Need the server scope serverScopeInfo = await this.EnsureSchemaAsync(cancellationToken, progress).ConfigureAwait(false); schema = serverScopeInfo.Schema; schema.EnsureSchema(); clientScope.Schema = schema; clientScope.Setup = serverScopeInfo.Setup; clientScope.Version = serverScopeInfo.Version; // generate a message to send var changesToSend = new HttpMessageSendChangesRequest(ctx, clientScope) { Changes = null, IsLastBatch = true, BatchIndex = 0 }; var serializer = this.SerializerFactory.GetSerializer <HttpMessageSendChangesRequest>(); var binaryData = await serializer.SerializeAsync(changesToSend); await this.InterceptAsync(new HttpMessageSendChangesRequestArgs(binaryData), cancellationToken).ConfigureAwait(false); // response var httpMessageContent = await this.httpRequestHandler.ProcessRequestAsync <HttpMessageSendChangesResponse> (this.HttpClient, this.ServiceUri, binaryData, HttpStep.GetChanges, ctx.SessionId, clientScope.Name, this.SerializerFactory, this.Converter, this.Options.BatchSize, cancellationToken).ConfigureAwait(false); // if nothing available, return empty response if (httpMessageContent.Changes == null) { return(httpMessageContent.RemoteClientTimestamp, null, new DatabaseChangesSelected()); } // Get if we need to work in memory or serialize things var workInMemoryLocally = this.Options.BatchSize == 0; bool isLastBatch; //timestamp generated by the server, hold in the client db long remoteClientTimestamp; // Create the BatchInfo and SyncContext to return at the end var serverBatchInfo = new BatchInfo(workInMemoryLocally, schema, this.Options.BatchDirectory); // stats DatabaseChangesSelected serverChangesSelected; // While we are not reaching the last batch from server do { // Check if we are at the last batch. // If so, we won't make another loop isLastBatch = httpMessageContent.IsLastBatch; ctx = httpMessageContent.SyncContext; remoteClientTimestamp = httpMessageContent.RemoteClientTimestamp; serverChangesSelected = httpMessageContent.ServerChangesSelected; var changesSet = serverBatchInfo.SanitizedSchema.Clone(); changesSet.ImportContainerSet(httpMessageContent.Changes, false); if (this.Converter != null && changesSet.HasRows) { AfterDeserializedRows(changesSet); } // Create a BatchPartInfo instance await serverBatchInfo.AddChangesAsync(changesSet, httpMessageContent.BatchIndex, isLastBatch, this); // free some memory if (httpMessageContent.Changes != null) { httpMessageContent.Changes.Clear(); } if (!isLastBatch) { // Ask for the next batch index var requestBatchIndex = httpMessageContent.BatchIndex + 1; // Create the message enveloppe var httpMessage = new HttpMessageGetMoreChangesRequest(ctx, requestBatchIndex); // serialize message var serializer2 = this.SerializerFactory.GetSerializer <HttpMessageGetMoreChangesRequest>(); var binaryData2 = await serializer2.SerializeAsync(httpMessage); await this.InterceptAsync(new HttpMessageGetMoreChangesRequestArgs(binaryData), cancellationToken).ConfigureAwait(false); httpMessageContent = await this.httpRequestHandler.ProcessRequestAsync <HttpMessageSendChangesResponse>( this.HttpClient, this.ServiceUri, binaryData2, HttpStep.GetMoreChanges, ctx.SessionId, this.ScopeName, this.SerializerFactory, this.Converter, 0, cancellationToken).ConfigureAwait(false); } } while (!isLastBatch); // generate the new scope item this.CompleteTime = DateTime.UtcNow; // Reaffect context this.SetContext(httpMessageContent.SyncContext); return(remoteClientTimestamp, serverBatchInfo, serverChangesSelected); }
GetSnapshotAsync(SyncSet schema = null, CancellationToken cancellationToken = default, IProgress <ProgressArgs> progress = null) { // Get context or create a new one var ctx = this.GetContext(); if (!this.StartTime.HasValue) { this.StartTime = DateTime.UtcNow; } // Make a remote call to get Schema from remote provider if (schema == null) { var serverScopeInfo = await this.EnsureSchemaAsync(cancellationToken, progress).ConfigureAwait(false); schema = serverScopeInfo.Schema; schema.EnsureSchema(); } // Create the BatchInfo and SyncContext to return at the end // Set InMemory by default to "true", but the real value is coming from server side var serverBatchInfo = new BatchInfo(false, schema, this.Options.BatchDirectory); bool isLastBatch; //timestamp generated by the server, hold in the client db long remoteClientTimestamp; // generate a message to send var changesToSend = new HttpMessageSendChangesRequest(ctx, null) { Changes = null, IsLastBatch = true, BatchIndex = 0 }; var serializer = this.SerializerFactory.GetSerializer <HttpMessageSendChangesRequest>(); var binaryData = await serializer.SerializeAsync(changesToSend); var httpMessageContent = await this.httpRequestHandler.ProcessRequestAsync <HttpMessageSendChangesResponse>( this.HttpClient, this.ServiceUri, binaryData, HttpStep.GetSnapshot, ctx.SessionId, this.ScopeName, this.SerializerFactory, this.Converter, 0, cancellationToken).ConfigureAwait(false); // if no snapshot available, return empty response if (httpMessageContent.Changes == null) { return(httpMessageContent.RemoteClientTimestamp, null); } // While we are not reaching the last batch from server do { // Check if we are at the last batch. // If so, we won't make another loop isLastBatch = httpMessageContent.IsLastBatch; ctx = httpMessageContent.SyncContext; remoteClientTimestamp = httpMessageContent.RemoteClientTimestamp; var changesSet = serverBatchInfo.SanitizedSchema.Clone(); changesSet.ImportContainerSet(httpMessageContent.Changes, false); if (this.Converter != null && changesSet.HasRows) { AfterDeserializedRows(changesSet); } // Create a BatchPartInfo instance await serverBatchInfo.AddChangesAsync(changesSet, httpMessageContent.BatchIndex, isLastBatch, this); // free some memory if (httpMessageContent.Changes != null) { httpMessageContent.Changes.Clear(); } if (!isLastBatch) { // Ask for the next batch index var requestBatchIndex = httpMessageContent.BatchIndex + 1; // Create the message enveloppe var httpMessage = new HttpMessageGetMoreChangesRequest(ctx, requestBatchIndex); // serialize message var serializer2 = this.SerializerFactory.GetSerializer <HttpMessageGetMoreChangesRequest>(); var binaryData2 = await serializer2.SerializeAsync(httpMessage); await this.InterceptAsync(new HttpMessageGetMoreChangesRequestArgs(binaryData), cancellationToken).ConfigureAwait(false); httpMessageContent = await this.httpRequestHandler.ProcessRequestAsync <HttpMessageSendChangesResponse>( this.HttpClient, this.ServiceUri, binaryData2, HttpStep.GetMoreChanges, ctx.SessionId, this.ScopeName, this.SerializerFactory, this.Converter, 0, cancellationToken).ConfigureAwait(false); } } while (!isLastBatch); // Reaffect context this.SetContext(httpMessageContent.SyncContext); return(remoteClientTimestamp, serverBatchInfo); }
ApplyThenGetChangesAsync(ScopeInfo scope, BatchInfo clientBatchInfo, CancellationToken cancellationToken = default, IProgress <ProgressArgs> progress = null) { SyncSet schema; // Get context or create a new one var ctx = this.GetContext(); if (!this.StartTime.HasValue) { this.StartTime = DateTime.UtcNow; } //// create the in memory changes set //var changesSet = new SyncSet(); // is it something that could happens ? if (scope.Schema == null) { // Make a remote call to get Schema from remote provider var serverScopeInfo = await this.EnsureSchemaAsync(cancellationToken, progress).ConfigureAwait(false); schema = serverScopeInfo.Schema; } else { schema = scope.Schema; } schema.EnsureSchema(); // if we don't have any BatchPartsInfo, just generate a new one to get, at least, something to send to the server // and get a response with new data from server if (clientBatchInfo == null) { clientBatchInfo = new BatchInfo(true, schema); } // Get sanitized schema, without readonly columns var sanitizedSchema = clientBatchInfo.SanitizedSchema; // -------------------------------------------------------------- // STEP 1 : Send everything to the server side // -------------------------------------------------------------- // response HttpMessageSendChangesResponse httpMessageContent = null; // If not in memory and BatchPartsInfo.Count == 0, nothing to send. // But we need to send something, so generate a little batch part if (clientBatchInfo.InMemory || (!clientBatchInfo.InMemory && clientBatchInfo.BatchPartsInfo.Count == 0)) { var changesToSend = new HttpMessageSendChangesRequest(ctx, scope); if (this.Converter != null && clientBatchInfo.InMemoryData != null && clientBatchInfo.InMemoryData.HasRows) { this.BeforeSerializeRows(clientBatchInfo.InMemoryData); } var containerSet = clientBatchInfo.InMemoryData == null ? new ContainerSet() : clientBatchInfo.InMemoryData.GetContainerSet(); changesToSend.Changes = containerSet; changesToSend.IsLastBatch = true; changesToSend.BatchIndex = 0; // serialize message var serializer = this.SerializerFactory.GetSerializer <HttpMessageSendChangesRequest>(); var binaryData = await serializer.SerializeAsync(changesToSend); await this.InterceptAsync(new HttpMessageSendChangesRequestArgs(binaryData), cancellationToken).ConfigureAwait(false); httpMessageContent = await this.httpRequestHandler.ProcessRequestAsync <HttpMessageSendChangesResponse> (this.HttpClient, this.ServiceUri, binaryData, HttpStep.SendChanges, ctx.SessionId, scope.Name, this.SerializerFactory, this.Converter, this.Options.BatchSize, cancellationToken).ConfigureAwait(false); } else { // Foreach part, will have to send them to the remote // once finished, return context foreach (var bpi in clientBatchInfo.BatchPartsInfo.OrderBy(bpi => bpi.Index)) { // If BPI is InMempory, no need to deserialize from disk // othewise load it await bpi.LoadBatchAsync(sanitizedSchema, clientBatchInfo.GetDirectoryFullPath(), this); var changesToSend = new HttpMessageSendChangesRequest(ctx, scope); if (this.Converter != null && bpi.Data.HasRows) { BeforeSerializeRows(bpi.Data); } // Set the change request properties changesToSend.Changes = bpi.Data.GetContainerSet(); changesToSend.IsLastBatch = bpi.IsLastBatch; changesToSend.BatchIndex = bpi.Index; // serialize message var serializer = this.SerializerFactory.GetSerializer <HttpMessageSendChangesRequest>(); var binaryData = await serializer.SerializeAsync(changesToSend); await this.InterceptAsync(new HttpMessageSendChangesRequestArgs(binaryData), cancellationToken).ConfigureAwait(false); httpMessageContent = await this.httpRequestHandler.ProcessRequestAsync <HttpMessageSendChangesResponse> (this.HttpClient, this.ServiceUri, binaryData, HttpStep.SendChanges, ctx.SessionId, scope.Name, this.SerializerFactory, this.Converter, this.Options.BatchSize, cancellationToken).ConfigureAwait(false); // for some reasons, if server don't want to wait for more, just break // That should never happened, actually if (httpMessageContent.ServerStep != HttpStep.SendChangesInProgress) { break; } } } // -------------------------------------------------------------- // STEP 2 : Receive everything from the server side // -------------------------------------------------------------- // Now we have sent all the datas to the server and now : // We have a FIRST response from the server with new datas // 1) Could be the only one response (enough or InMemory is set on the server side) // 2) Could bt the first response and we need to download all batchs // While we have an other batch to process var isLastBatch = false; // Get if we need to work in memory or serialize things var workInMemoryLocally = this.Options.BatchSize == 0; // Create the BatchInfo and SyncContext to return at the end // Set InMemory by default to "true", but the real value is coming from server side var serverBatchInfo = new BatchInfo(workInMemoryLocally, schema, this.Options.BatchDirectory); // stats DatabaseChangesSelected serverChangesSelected = null; DatabaseChangesApplied clientChangesApplied = null; //timestamp generated by the server, hold in the client db long remoteClientTimestamp = 0; // While we are not reaching the last batch from server do { // Check if we are at the last batch. // If so, we won't make another loop isLastBatch = httpMessageContent.IsLastBatch; serverChangesSelected = httpMessageContent.ServerChangesSelected; clientChangesApplied = httpMessageContent.ClientChangesApplied; ctx = httpMessageContent.SyncContext; remoteClientTimestamp = httpMessageContent.RemoteClientTimestamp; var changesSet = sanitizedSchema.Clone(); changesSet.ImportContainerSet(httpMessageContent.Changes, false); if (this.Converter != null && changesSet.HasRows) { AfterDeserializedRows(changesSet); } // Create a BatchPartInfo instance await serverBatchInfo.AddChangesAsync(changesSet, httpMessageContent.BatchIndex, isLastBatch, this); // free some memory if (!workInMemoryLocally && httpMessageContent.Changes != null) { httpMessageContent.Changes.Clear(); } if (!isLastBatch) { // Ask for the next batch index var requestBatchIndex = httpMessageContent.BatchIndex + 1; // Create the message enveloppe var httpMessage = new HttpMessageGetMoreChangesRequest(ctx, requestBatchIndex); // serialize message var serializer = this.SerializerFactory.GetSerializer <HttpMessageGetMoreChangesRequest>(); var binaryData = await serializer.SerializeAsync(httpMessage); await this.InterceptAsync(new HttpMessageGetMoreChangesRequestArgs(binaryData), cancellationToken).ConfigureAwait(false); httpMessageContent = await this.httpRequestHandler.ProcessRequestAsync <HttpMessageSendChangesResponse>( this.HttpClient, this.ServiceUri, binaryData, HttpStep.GetMoreChanges, ctx.SessionId, scope.Name, this.SerializerFactory, this.Converter, this.Options.BatchSize, cancellationToken).ConfigureAwait(false); } } while (!isLastBatch); // generate the new scope item this.CompleteTime = DateTime.UtcNow; // Reaffect context this.SetContext(httpMessageContent.SyncContext); return(remoteClientTimestamp, serverBatchInfo, httpMessageContent.ConflictResolutionPolicy, clientChangesApplied, serverChangesSelected); }
/// <summary> /// Gets a batch of changes to synchronize when given batch size, /// destination knowledge, and change data retriever parameters. /// </summary> /// <returns>A DbSyncContext object that will be used to retrieve the modified data.</returns> public virtual async Task <(SyncContext, BatchInfo, DatabaseChangesSelected)> GetChangeBatchAsync( SyncContext context, MessageGetChangesBatch message, DbConnection connection, DbTransaction transaction, CancellationToken cancellationToken, IProgress <ProgressArgs> progress = null) { // batch info containing changes BatchInfo batchInfo; // Statistics about changes that are selected DatabaseChangesSelected changesSelected; if (context.SyncWay == SyncWay.Upload && context.SyncType == SyncType.Reinitialize) { (batchInfo, changesSelected) = await this.GetEmptyChangesAsync(message); return(context, batchInfo, changesSelected); } // Check if the provider is not outdated var isOutdated = this.IsRemoteOutdated(); // Get a chance to make the sync even if it's outdated if (isOutdated) { var outdatedArgs = new OutdatedArgs(context, null, null); // Interceptor await this.InterceptAsync(outdatedArgs).ConfigureAwait(false); if (outdatedArgs.Action != OutdatedAction.Rollback) { context.SyncType = outdatedArgs.Action == OutdatedAction.Reinitialize ? SyncType.Reinitialize : SyncType.ReinitializeWithUpload; } if (outdatedArgs.Action == OutdatedAction.Rollback) { throw new OutOfDateException(); } } // create local directory if (message.BatchSize > 0 && !string.IsNullOrEmpty(message.BatchDirectory) && !Directory.Exists(message.BatchDirectory)) { Directory.CreateDirectory(message.BatchDirectory); } // numbers of batch files generated var batchIndex = 0; // Check if we are in batch mode var isBatch = message.BatchSize > 0; // Create stats object to store changes count var changes = new DatabaseChangesSelected(); // create the in memory changes set var changesSet = new SyncSet(message.Schema.ScopeName); // Create a Schema set without readonly columns, attached to memory changes foreach (var table in message.Schema.Tables) { DbSyncAdapter.CreateChangesTable(message.Schema.Tables[table.TableName, table.SchemaName], changesSet); } // Create a batch info in memory (if !isBatch) or serialized on disk (if isBatch) // batchinfo generate a schema clone with scope columns if needed batchInfo = new BatchInfo(!isBatch, changesSet, message.BatchDirectory); // Clear tables, we will add only the ones we need in the batch info changesSet.Clear(); foreach (var syncTable in message.Schema.Tables) { // if we are in upload stage, so check if table is not download only if (context.SyncWay == SyncWay.Upload && syncTable.SyncDirection == SyncDirection.DownloadOnly) { continue; } // if we are in download stage, so check if table is not download only if (context.SyncWay == SyncWay.Download && syncTable.SyncDirection == SyncDirection.UploadOnly) { continue; } var tableBuilder = this.GetTableBuilder(syncTable); var syncAdapter = tableBuilder.CreateSyncAdapter(connection, transaction); // raise before event context.SyncStage = SyncStage.TableChangesSelecting; var tableChangesSelectingArgs = new TableChangesSelectingArgs(context, syncTable.TableName, connection, transaction); // launch interceptor if any await this.InterceptAsync(tableChangesSelectingArgs).ConfigureAwait(false); // Get Command var selectIncrementalChangesCommand = this.GetSelectChangesCommand(context, syncAdapter, syncTable, message.IsNew); // Set parameters this.SetSelectChangesCommonParameters(context, syncTable, message.ExcludingScopeId, message.IsNew, message.LastTimestamp, selectIncrementalChangesCommand); // Statistics var tableChangesSelected = new TableChangesSelected(syncTable.TableName); // Get the reader using (var dataReader = selectIncrementalChangesCommand.ExecuteReader()) { // memory size total double rowsMemorySize = 0L; // Create a chnages table with scope columns var changesSetTable = DbSyncAdapter.CreateChangesTable(message.Schema.Tables[syncTable.TableName, syncTable.SchemaName], changesSet); while (dataReader.Read()) { // Create a row from dataReader var row = CreateSyncRowFromReader(dataReader, changesSetTable); // Add the row to the changes set changesSetTable.Rows.Add(row); // Set the correct state to be applied if (row.RowState == DataRowState.Deleted) { tableChangesSelected.Deletes++; } else if (row.RowState == DataRowState.Modified) { tableChangesSelected.Upserts++; } // calculate row size if in batch mode if (isBatch) { var fieldsSize = ContainerTable.GetRowSizeFromDataRow(row.ToArray()); var finalFieldSize = fieldsSize / 1024d; if (finalFieldSize > message.BatchSize) { throw new RowOverSizedException(finalFieldSize.ToString()); } // Calculate the new memory size rowsMemorySize += finalFieldSize; // Next line if we don't reach the batch size yet. if (rowsMemorySize <= message.BatchSize) { continue; } // add changes to batchinfo await batchInfo.AddChangesAsync(changesSet, batchIndex, false); // increment batch index batchIndex++; // we know the datas are serialized here, so we can flush the set changesSet.Clear(); // Recreate an empty ContainerSet and a ContainerTable changesSet = new SyncSet(message.Schema.ScopeName); changesSetTable = DbSyncAdapter.CreateChangesTable(message.Schema.Tables[syncTable.TableName, syncTable.SchemaName], changesSet); // Init the row memory size rowsMemorySize = 0L; } } } selectIncrementalChangesCommand.Dispose(); context.SyncStage = SyncStage.TableChangesSelected; if (tableChangesSelected.Deletes > 0 || tableChangesSelected.Upserts > 0) { changes.TableChangesSelected.Add(tableChangesSelected); } // Event progress & interceptor context.SyncStage = SyncStage.TableChangesSelected; var tableChangesSelectedArgs = new TableChangesSelectedArgs(context, tableChangesSelected, connection, transaction); this.ReportProgress(context, progress, tableChangesSelectedArgs); await this.InterceptAsync(tableChangesSelectedArgs).ConfigureAwait(false); } // We are in batch mode, and we are at the last batchpart info // Even if we don't have rows inside, we return the changesSet, since it contains at leaset schema if (changesSet != null && changesSet.HasTables) { await batchInfo.AddChangesAsync(changesSet, batchIndex, true).ConfigureAwait(false); } // Check the last index as the last batch batchInfo.EnsureLastBatch(); return(context, batchInfo, changes); }