InternalApplyThenGetChangesAsync(ClientScopeInfo clientScopeInfo, SyncContext context, BatchInfo clientBatchInfo, DbConnection connection = default, DbTransaction transaction = default, CancellationToken cancellationToken = default, IProgress <ProgressArgs> progress = null) { await using var runner = await this.GetConnectionAsync(context, SyncMode.Reading, SyncStage.ChangesApplying, connection, transaction, cancellationToken, progress).ConfigureAwait(false); SyncSet schema; ServerScopeInfo serverScopeInfo; // is it something that could happens ? if (clientScopeInfo.Schema == null) { // Make a remote call to get Schema from remote provider (context, serverScopeInfo) = await this.InternalGetServerScopeInfoAsync( context, null, runner.Connection, runner.Transaction, runner.CancellationToken, runner.Progress).ConfigureAwait(false); schema = serverScopeInfo.Schema; } else { schema = clientScopeInfo.Schema; } schema.EnsureSchema(); // if we don't have any BatchPartsInfo, just generate a new one to get, at least, something to send to the server // and get a response with new data from server if (clientBatchInfo == null) { clientBatchInfo = new BatchInfo(schema); } // -------------------------------------------------------------- // STEP 1 : Send everything to the server side // -------------------------------------------------------------- HttpResponseMessage response = null; // If not in memory and BatchPartsInfo.Count == 0, nothing to send. // But we need to send something, so generate a little batch part if (clientBatchInfo.BatchPartsInfo.Count == 0) { var changesToSend = new HttpMessageSendChangesRequest(context, clientScopeInfo); var containerSet = new ContainerSet(); changesToSend.Changes = containerSet; changesToSend.IsLastBatch = true; changesToSend.BatchIndex = 0; changesToSend.BatchCount = clientBatchInfo.BatchPartsInfo == null ? 0 : clientBatchInfo.BatchPartsInfo.Count; var inMemoryRowsCount = changesToSend.Changes.RowsCount(); context.ProgressPercentage += 0.125; await this.InterceptAsync(new HttpSendingClientChangesRequestArgs(changesToSend, inMemoryRowsCount, inMemoryRowsCount, this.GetServiceHost()), progress, cancellationToken).ConfigureAwait(false); // serialize message var serializer = this.SerializerFactory.GetSerializer <HttpMessageSendChangesRequest>(); var binaryData = await serializer.SerializeAsync(changesToSend); response = await this.httpRequestHandler.ProcessRequestAsync (this.HttpClient, context, this.ServiceUri, binaryData, HttpStep.SendChangesInProgress, this.SerializerFactory, this.Converter, this.Options.BatchSize, this.SyncPolicy, cancellationToken, progress).ConfigureAwait(false); } else { int tmpRowsSendedCount = 0; // Foreach part, will have to send them to the remote // once finished, return context var initialPctProgress1 = context.ProgressPercentage; var localSerializer = new LocalJsonSerializer(); var interceptorsReading = this.interceptors.GetInterceptors <DeserializingRowArgs>(); if (interceptorsReading.Count > 0) { localSerializer.OnReadingRow(async(schemaTable, rowString) => { var args = new DeserializingRowArgs(context, schemaTable, rowString); await this.InterceptAsync(args); return(args.Result); }); } foreach (var bpi in clientBatchInfo.BatchPartsInfo.OrderBy(bpi => bpi.Index)) { // Get the updatable schema for the only table contained in the batchpartinfo var schemaTable = DbSyncAdapter.CreateChangesTable(schema.Tables[bpi.Tables[0].TableName, bpi.Tables[0].SchemaName]); // Generate the ContainerSet containing rows to send to the user var containerSet = new ContainerSet(); var containerTable = new ContainerTable(schemaTable); var fullPath = Path.Combine(clientBatchInfo.GetDirectoryFullPath(), bpi.FileName); containerSet.Tables.Add(containerTable); // read rows from file foreach (var row in localSerializer.ReadRowsFromFile(fullPath, schemaTable)) { containerTable.Rows.Add(row.ToArray()); } // Call the converter if needed if (this.Converter != null && containerTable.HasRows) { BeforeSerializeRows(containerTable, schemaTable, this.Converter); } // Create the send changes request var changesToSend = new HttpMessageSendChangesRequest(context, clientScopeInfo) { Changes = containerSet, IsLastBatch = bpi.IsLastBatch, BatchIndex = bpi.Index, BatchCount = clientBatchInfo.BatchPartsInfo.Count }; tmpRowsSendedCount += containerTable.Rows.Count; context.ProgressPercentage = initialPctProgress1 + ((changesToSend.BatchIndex + 1) * 0.2d / changesToSend.BatchCount); await this.InterceptAsync(new HttpSendingClientChangesRequestArgs(changesToSend, tmpRowsSendedCount, clientBatchInfo.RowsCount, this.GetServiceHost()), progress, cancellationToken).ConfigureAwait(false); // serialize message var serializer = this.SerializerFactory.GetSerializer <HttpMessageSendChangesRequest>(); var binaryData = await serializer.SerializeAsync(changesToSend); response = await this.httpRequestHandler.ProcessRequestAsync (this.HttpClient, context, this.ServiceUri, binaryData, HttpStep.SendChangesInProgress, this.SerializerFactory, this.Converter, this.Options.BatchSize, this.SyncPolicy, cancellationToken, progress).ConfigureAwait(false); // See #721 for issue and #721 for PR from slagtejn if (!bpi.IsLastBatch) { response.Dispose(); } } } // -------------------------------------------------------------- // STEP 2 : Receive everything from the server side // -------------------------------------------------------------- // Now we have sent all the datas to the server and now : // We have a FIRST response from the server with new datas // 1) Could be the only one response // 2) Could be the first response and we need to download all batchs context.SyncStage = SyncStage.ChangesSelecting; var initialPctProgress = 0.55; context.ProgressPercentage = initialPctProgress; // Create the BatchInfo var serverBatchInfo = new BatchInfo(schema); HttpMessageSummaryResponse summaryResponseContent = null; // Deserialize response incoming from server using (var streamResponse = await response.Content.ReadAsStreamAsync().ConfigureAwait(false)) { var responseSerializer = this.SerializerFactory.GetSerializer <HttpMessageSummaryResponse>(); summaryResponseContent = await responseSerializer.DeserializeAsync(streamResponse); } serverBatchInfo.RowsCount = summaryResponseContent.BatchInfo.RowsCount; serverBatchInfo.Timestamp = summaryResponseContent.RemoteClientTimestamp; context = summaryResponseContent.SyncContext; if (summaryResponseContent.BatchInfo.BatchPartsInfo != null) { foreach (var bpi in summaryResponseContent.BatchInfo.BatchPartsInfo) { serverBatchInfo.BatchPartsInfo.Add(bpi); } } // From here, we need to serialize everything on disk // Generate the batch directory var batchDirectoryRoot = this.Options.BatchDirectory; var batchDirectoryName = string.Concat(DateTime.UtcNow.ToString("yyyy_MM_dd_ss"), Path.GetRandomFileName().Replace(".", "")); serverBatchInfo.DirectoryRoot = batchDirectoryRoot; serverBatchInfo.DirectoryName = batchDirectoryName; if (!Directory.Exists(serverBatchInfo.GetDirectoryFullPath())) { Directory.CreateDirectory(serverBatchInfo.GetDirectoryFullPath()); } // If we have a snapshot we are raising the batches downloading process that will occurs await this.InterceptAsync(new HttpBatchesDownloadingArgs(context, serverBatchInfo, this.GetServiceHost()), progress, cancellationToken).ConfigureAwait(false); // function used to download one part var dl = new Func <BatchPartInfo, Task>(async(bpi) => { if (cancellationToken.IsCancellationRequested) { return; } var changesToSend3 = new HttpMessageGetMoreChangesRequest(context, bpi.Index); var serializer3 = this.SerializerFactory.GetSerializer <HttpMessageGetMoreChangesRequest>(); var binaryData3 = await serializer3.SerializeAsync(changesToSend3).ConfigureAwait(false); var step3 = HttpStep.GetMoreChanges; await this.InterceptAsync(new HttpGettingServerChangesRequestArgs(bpi.Index, serverBatchInfo.BatchPartsInfo.Count, summaryResponseContent.SyncContext, this.GetServiceHost()), progress, cancellationToken).ConfigureAwait(false); // Raise get changes request context.ProgressPercentage = initialPctProgress + ((bpi.Index + 1) * 0.2d / serverBatchInfo.BatchPartsInfo.Count); var response = await this.httpRequestHandler.ProcessRequestAsync( this.HttpClient, context, this.ServiceUri, binaryData3, step3, this.SerializerFactory, this.Converter, 0, this.SyncPolicy, cancellationToken, progress).ConfigureAwait(false); if (this.SerializerFactory.Key != "json") { var webSerializer = this.SerializerFactory.GetSerializer <HttpMessageSendChangesResponse>(); using var responseStream = await response.Content.ReadAsStreamAsync().ConfigureAwait(false); var getMoreChanges = await webSerializer.DeserializeAsync(responseStream); context = getMoreChanges.SyncContext; if (getMoreChanges != null && getMoreChanges.Changes != null && getMoreChanges.Changes.HasRows) { var localSerializer = new LocalJsonSerializer(); var interceptorsWriting = this.interceptors.GetInterceptors <SerializingRowArgs>(); if (interceptorsWriting.Count > 0) { localSerializer.OnWritingRow(async(syncTable, rowArray) => { var args = new SerializingRowArgs(context, syncTable, rowArray); await this.InterceptAsync(args, progress, cancellationToken).ConfigureAwait(false); return(args.Result); }); } // Should have only one table var table = getMoreChanges.Changes.Tables[0]; var schemaTable = DbSyncAdapter.CreateChangesTable(schema.Tables[table.TableName, table.SchemaName]); var fullPath = Path.Combine(serverBatchInfo.GetDirectoryFullPath(), bpi.FileName); // open the file and write table header await localSerializer.OpenFileAsync(fullPath, schemaTable).ConfigureAwait(false); foreach (var row in table.Rows) { await localSerializer.WriteRowToFileAsync(new SyncRow(schemaTable, row), schemaTable).ConfigureAwait(false); } // Close file await localSerializer.CloseFileAsync(fullPath, schemaTable).ConfigureAwait(false); } } else { // Serialize await SerializeAsync(response, bpi.FileName, serverBatchInfo.GetDirectoryFullPath(), this).ConfigureAwait(false); } // Raise response from server containing a batch changes await this.InterceptAsync(new HttpGettingServerChangesResponseArgs(serverBatchInfo, bpi.Index, bpi.RowsCount, summaryResponseContent.SyncContext, this.GetServiceHost()), progress, cancellationToken).ConfigureAwait(false); }); // Parrallel download of all bpis (which will launch the delete directory on the server side) await serverBatchInfo.BatchPartsInfo.ForEachAsync(bpi => dl(bpi), this.MaxDownladingDegreeOfParallelism).ConfigureAwait(false); // Send order of end of download var lastBpi = serverBatchInfo.BatchPartsInfo.FirstOrDefault(bpi => bpi.IsLastBatch); if (lastBpi != null) { var endOfDownloadChanges = new HttpMessageGetMoreChangesRequest(context, lastBpi.Index); var serializerEndOfDownloadChanges = this.SerializerFactory.GetSerializer <HttpMessageGetMoreChangesRequest>(); var binaryData3 = await serializerEndOfDownloadChanges.SerializeAsync(endOfDownloadChanges).ConfigureAwait(false); var endResponse = await this.httpRequestHandler.ProcessRequestAsync( this.HttpClient, context, this.ServiceUri, binaryData3, HttpStep.SendEndDownloadChanges, this.SerializerFactory, this.Converter, 0, this.SyncPolicy, cancellationToken, progress).ConfigureAwait(false); // Deserialize response incoming from server // This is the last response // Should contains step HttpStep.SendEndDownloadChanges using var streamResponse = await endResponse.Content.ReadAsStreamAsync().ConfigureAwait(false); var endResponseSerializer = this.SerializerFactory.GetSerializer <HttpMessageSendChangesResponse>(); var endResponseContent = await endResponseSerializer.DeserializeAsync(streamResponse); context = endResponseContent.SyncContext; } // generate the new scope item this.CompleteTime = DateTime.UtcNow; await this.InterceptAsync(new HttpBatchesDownloadedArgs(summaryResponseContent, summaryResponseContent.SyncContext, this.GetServiceHost()), progress, cancellationToken).ConfigureAwait(false); var serverSyncChanges = new ServerSyncChanges( summaryResponseContent.RemoteClientTimestamp, serverBatchInfo, summaryResponseContent.ServerChangesSelected ); return(context, serverSyncChanges, summaryResponseContent.ClientChangesApplied, summaryResponseContent.ConflictResolutionPolicy); }
/// <summary> /// Create a response message content based on a requested index in a server batch info /// </summary> private HttpMessageSendChangesResponse GetChangesResponse(SyncContext syncContext, long remoteClientTimestamp, BatchInfo serverBatchInfo, DatabaseChangesSelected serverChangesSelected, int batchIndexRequested, ConflictResolutionPolicy policy) { // 1) Create the http message content response var changesResponse = new HttpMessageSendChangesResponse(syncContext); changesResponse.ChangesSelected = serverChangesSelected; changesResponse.ServerStep = HttpStep.GetChanges; changesResponse.ConflictResolutionPolicy = policy; // If nothing to do, just send back if (serverBatchInfo.InMemory || serverBatchInfo.BatchPartsInfo.Count == 0) { if (this.ClientConverter != null && serverBatchInfo.InMemoryData.HasRows) { BeforeSerializeRows(serverBatchInfo.InMemoryData, this.ClientConverter); } changesResponse.Changes = serverBatchInfo.InMemoryData.GetContainerSet(); changesResponse.BatchIndex = 0; changesResponse.IsLastBatch = true; changesResponse.RemoteClientTimestamp = remoteClientTimestamp; return(changesResponse); } // Get the batch part index requested var batchPartInfo = serverBatchInfo.BatchPartsInfo.First(d => d.Index == batchIndexRequested); // if we are not in memory, we set the BI in session, to be able to get it back on next request // create the in memory changes set var changesSet = new SyncSet(Schema.ScopeName); foreach (var table in Schema.Tables) { DbSyncAdapter.CreateChangesTable(Schema.Tables[table.TableName, table.SchemaName], changesSet); } batchPartInfo.LoadBatch(changesSet, serverBatchInfo.GetDirectoryFullPath()); // if client request a conversion on each row, apply the conversion if (this.ClientConverter != null && batchPartInfo.Data.HasRows) { BeforeSerializeRows(batchPartInfo.Data, this.ClientConverter); } changesResponse.Changes = batchPartInfo.Data.GetContainerSet(); changesResponse.BatchIndex = batchIndexRequested; changesResponse.IsLastBatch = batchPartInfo.IsLastBatch; changesResponse.RemoteClientTimestamp = remoteClientTimestamp; changesResponse.ServerStep = batchPartInfo.IsLastBatch ? HttpStep.GetChanges : HttpStep.GetChangesInProgress; // If we have only one bpi, we can safely delete it if (batchPartInfo.IsLastBatch) { // delete the folder (not the BatchPartInfo, because we have a reference on it) if (this.Options.CleanFolder) { var shouldDeleteFolder = true; if (!string.IsNullOrEmpty(this.Options.SnapshotsDirectory)) { var dirInfo = new DirectoryInfo(serverBatchInfo.DirectoryRoot); var snapInfo = new DirectoryInfo(this.Options.SnapshotsDirectory); shouldDeleteFolder = dirInfo.FullName != snapInfo.FullName; } if (shouldDeleteFolder) { serverBatchInfo.TryRemoveDirectory(); } } } return(changesResponse); }
GetChangesAsync(ClientScopeInfo clientScopeInfo, SyncParameters parameters = default, DbConnection connection = default, DbTransaction transaction = default, CancellationToken cancellationToken = default, IProgress <ProgressArgs> progress = null) { var context = new SyncContext(Guid.NewGuid(), clientScopeInfo.Name); if (parameters != null) { context.Parameters = parameters; } SyncSet schema; ServerScopeInfo serverScopeInfo; // Need the server scope (context, serverScopeInfo) = await this.InternalGetServerScopeInfoAsync(context, clientScopeInfo.Setup, connection, transaction, cancellationToken, progress).ConfigureAwait(false); schema = serverScopeInfo.Schema; schema.EnsureSchema(); clientScopeInfo.Schema = schema; clientScopeInfo.Setup = serverScopeInfo.Setup; clientScopeInfo.Version = serverScopeInfo.Version; var changesToSend = new HttpMessageSendChangesRequest(context, clientScopeInfo); var containerSet = new ContainerSet(); changesToSend.Changes = containerSet; changesToSend.IsLastBatch = true; changesToSend.BatchIndex = 0; changesToSend.BatchCount = 0; context.ProgressPercentage += 0.125; await this.InterceptAsync(new HttpSendingClientChangesRequestArgs(changesToSend, 0, 0, this.GetServiceHost()), progress, cancellationToken).ConfigureAwait(false); // serialize message var serializer = this.SerializerFactory.GetSerializer <HttpMessageSendChangesRequest>(); var binaryData = await serializer.SerializeAsync(changesToSend); var response = await this.httpRequestHandler.ProcessRequestAsync (this.HttpClient, context, this.ServiceUri, binaryData, HttpStep.SendChangesInProgress, this.SerializerFactory, this.Converter, this.Options.BatchSize, this.SyncPolicy, cancellationToken, progress).ConfigureAwait(false); // -------------------------------------------------------------- // STEP 2 : Receive everything from the server side // -------------------------------------------------------------- // Now we have sent all the datas to the server and now : // We have a FIRST response from the server with new datas // 1) Could be the only one response (enough or InMemory is set on the server side) // 2) Could bt the first response and we need to download all batchs context.SyncStage = SyncStage.ChangesSelecting; var initialPctProgress = 0.55; context.ProgressPercentage = initialPctProgress; // Create the BatchInfo var serverBatchInfo = new BatchInfo(schema); HttpMessageSummaryResponse summaryResponseContent = null; // Deserialize response incoming from server using (var streamResponse = await response.Content.ReadAsStreamAsync().ConfigureAwait(false)) { var responseSerializer = this.SerializerFactory.GetSerializer <HttpMessageSummaryResponse>(); summaryResponseContent = await responseSerializer.DeserializeAsync(streamResponse); } serverBatchInfo.RowsCount = summaryResponseContent.BatchInfo.RowsCount; serverBatchInfo.Timestamp = summaryResponseContent.RemoteClientTimestamp; context = summaryResponseContent.SyncContext; if (summaryResponseContent.BatchInfo.BatchPartsInfo != null) { foreach (var bpi in summaryResponseContent.BatchInfo.BatchPartsInfo) { serverBatchInfo.BatchPartsInfo.Add(bpi); } } //----------------------- // In Batch Mode //----------------------- // From here, we need to serialize everything on disk // Generate the batch directory var batchDirectoryRoot = this.Options.BatchDirectory; var batchDirectoryName = string.Concat(DateTime.UtcNow.ToString("yyyy_MM_dd_ss"), Path.GetRandomFileName().Replace(".", "")); serverBatchInfo.DirectoryRoot = batchDirectoryRoot; serverBatchInfo.DirectoryName = batchDirectoryName; if (!Directory.Exists(serverBatchInfo.GetDirectoryFullPath())) { Directory.CreateDirectory(serverBatchInfo.GetDirectoryFullPath()); } // hook to get the last batch part info at the end var bpis = serverBatchInfo.BatchPartsInfo.Where(bpi => !bpi.IsLastBatch); var lstbpi = serverBatchInfo.BatchPartsInfo.First(bpi => bpi.IsLastBatch); // function used to download one part var dl = new Func <BatchPartInfo, Task>(async(bpi) => { var changesToSend3 = new HttpMessageGetMoreChangesRequest(context, bpi.Index); var serializer3 = this.SerializerFactory.GetSerializer <HttpMessageGetMoreChangesRequest>(); var binaryData3 = await serializer3.SerializeAsync(changesToSend3).ConfigureAwait(false); var step3 = HttpStep.GetMoreChanges; await this.InterceptAsync(new HttpGettingServerChangesRequestArgs(bpi.Index, serverBatchInfo.BatchPartsInfo.Count, summaryResponseContent.SyncContext, this.GetServiceHost()), progress, cancellationToken).ConfigureAwait(false); // Raise get changes request context.ProgressPercentage = initialPctProgress + ((bpi.Index + 1) * 0.2d / serverBatchInfo.BatchPartsInfo.Count); var response = await this.httpRequestHandler.ProcessRequestAsync( this.HttpClient, context, this.ServiceUri, binaryData3, step3, this.SerializerFactory, this.Converter, 0, this.SyncPolicy, cancellationToken, progress).ConfigureAwait(false); // Serialize await SerializeAsync(response, bpi.FileName, serverBatchInfo.GetDirectoryFullPath(), this).ConfigureAwait(false); // Raise response from server containing a batch changes await this.InterceptAsync(new HttpGettingServerChangesResponseArgs(serverBatchInfo, bpi.Index, bpi.RowsCount, summaryResponseContent.SyncContext, this.GetServiceHost()), progress, cancellationToken).ConfigureAwait(false); response.Dispose(); }); // Parrallel download of all bpis except the last one (which will launch the delete directory on the server side) await bpis.ForEachAsync(bpi => dl(bpi), this.MaxDownladingDegreeOfParallelism).ConfigureAwait(false); // Download last batch part that will launch the server deletion of the tmp dir await dl(lstbpi).ConfigureAwait(false); // generate the new scope item this.CompleteTime = DateTime.UtcNow; // Reaffect context context = summaryResponseContent.SyncContext; return(new ServerSyncChanges(summaryResponseContent.RemoteClientTimestamp, serverBatchInfo, summaryResponseContent.ServerChangesSelected)); }
/// <summary> /// Gets a batch of changes to synchronize when given batch size, /// destination knowledge, and change data retriever parameters. /// </summary> /// <returns>A DbSyncContext object that will be used to retrieve the modified data.</returns> internal virtual async Task <(SyncContext, BatchInfo, DatabaseChangesSelected)> InternalGetChangesAsync( IScopeInfo scopeInfo, SyncContext context, bool isNew, long?fromLastTimestamp, long?toNewTimestamp, Guid?excludingScopeId, bool supportsMultiActiveResultSets, string batchRootDirectory, string batchDirectoryName, DbConnection connection, DbTransaction transaction, CancellationToken cancellationToken, IProgress <ProgressArgs> progress) { // batch info containing changes BatchInfo batchInfo; // Statistics about changes that are selected DatabaseChangesSelected changesSelected; context.SyncStage = SyncStage.ChangesSelecting; if (context.SyncWay == SyncWay.Upload && context.SyncType == SyncType.Reinitialize) { (batchInfo, changesSelected) = await this.InternalGetEmptyChangesAsync(scopeInfo, batchRootDirectory).ConfigureAwait(false); return(context, batchInfo, changesSelected); } // create local directory if (!string.IsNullOrEmpty(batchRootDirectory) && !Directory.Exists(batchRootDirectory)) { Directory.CreateDirectory(batchRootDirectory); } changesSelected = new DatabaseChangesSelected(); // Create a batch // batchinfo generate a schema clone with scope columns if needed batchInfo = new BatchInfo(scopeInfo.Schema, batchRootDirectory, batchDirectoryName); batchInfo.TryRemoveDirectory(); batchInfo.CreateDirectory(); // Call interceptor var databaseChangesSelectingArgs = new DatabaseChangesSelectingArgs(context, batchInfo.GetDirectoryFullPath(), this.Options.BatchSize, isNew, fromLastTimestamp, toNewTimestamp, connection, transaction); await this.InterceptAsync(databaseChangesSelectingArgs, progress, cancellationToken).ConfigureAwait(false); var cptSyncTable = 0; var currentProgress = context.ProgressPercentage; var schemaTables = scopeInfo.Schema.Tables.SortByDependencies(tab => tab.GetRelations().Select(r => r.GetParentTable())); var lstAllBatchPartInfos = new ConcurrentBag <BatchPartInfo>(); var lstTableChangesSelected = new ConcurrentBag <TableChangesSelected>(); var threadNumberLimits = supportsMultiActiveResultSets ? 16 : 1; if (supportsMultiActiveResultSets) { await schemaTables.ForEachAsync(async syncTable => { if (cancellationToken.IsCancellationRequested) { return; } // tmp count of table for report progress pct cptSyncTable++; List <BatchPartInfo> syncTableBatchPartInfos; TableChangesSelected tableChangesSelected; (context, syncTableBatchPartInfos, tableChangesSelected) = await InternalReadSyncTableChangesAsync( scopeInfo, context, excludingScopeId, syncTable, batchInfo, isNew, fromLastTimestamp, connection, transaction, cancellationToken, progress).ConfigureAwait(false); if (syncTableBatchPartInfos == null) { return; } // We don't report progress if no table changes is empty, to limit verbosity if (tableChangesSelected != null && (tableChangesSelected.Deletes > 0 || tableChangesSelected.Upserts > 0)) { lstTableChangesSelected.Add(tableChangesSelected); } // Add sync table bpi to all bpi syncTableBatchPartInfos.ForEach(bpi => lstAllBatchPartInfos.Add(bpi)); context.ProgressPercentage = currentProgress + (cptSyncTable * 0.2d / scopeInfo.Schema.Tables.Count); }, threadNumberLimits); } else { foreach (var syncTable in schemaTables) { if (cancellationToken.IsCancellationRequested) { continue; } // tmp count of table for report progress pct cptSyncTable++; List <BatchPartInfo> syncTableBatchPartInfos; TableChangesSelected tableChangesSelected; (context, syncTableBatchPartInfos, tableChangesSelected) = await InternalReadSyncTableChangesAsync( scopeInfo, context, excludingScopeId, syncTable, batchInfo, isNew, fromLastTimestamp, connection, transaction, cancellationToken, progress).ConfigureAwait(false); if (syncTableBatchPartInfos == null) { continue; } // We don't report progress if no table changes is empty, to limit verbosity if (tableChangesSelected != null && (tableChangesSelected.Deletes > 0 || tableChangesSelected.Upserts > 0)) { lstTableChangesSelected.Add(tableChangesSelected); } // Add sync table bpi to all bpi syncTableBatchPartInfos.ForEach(bpi => lstAllBatchPartInfos.Add(bpi)); context.ProgressPercentage = currentProgress + (cptSyncTable * 0.2d / scopeInfo.Schema.Tables.Count); } } while (!lstTableChangesSelected.IsEmpty) { if (lstTableChangesSelected.TryTake(out var tableChangesSelected)) { changesSelected.TableChangesSelected.Add(tableChangesSelected); } } // delete all empty batchparts (empty tables) foreach (var bpi in lstAllBatchPartInfos.Where(bpi => bpi.RowsCount <= 0)) { File.Delete(Path.Combine(batchInfo.GetDirectoryFullPath(), bpi.FileName)); } // Generate a good index order to be compliant with previous versions var tmpLstBatchPartInfos = new List <BatchPartInfo>(); foreach (var table in schemaTables) { // get all bpi where count > 0 and ordered by index foreach (var bpi in lstAllBatchPartInfos.Where(bpi => bpi.RowsCount > 0 && bpi.Tables[0].EqualsByName(new BatchPartTableInfo(table.TableName, table.SchemaName))).OrderBy(bpi => bpi.Index).ToArray()) { batchInfo.BatchPartsInfo.Add(bpi); batchInfo.RowsCount += bpi.RowsCount; tmpLstBatchPartInfos.Add(bpi); } } var newBatchIndex = 0; foreach (var bpi in tmpLstBatchPartInfos) { bpi.Index = newBatchIndex; newBatchIndex++; bpi.IsLastBatch = newBatchIndex == tmpLstBatchPartInfos.Count; } //Set the total rows count contained in the batch info batchInfo.EnsureLastBatch(); if (batchInfo.RowsCount <= 0) { var cleanFolder = await this.InternalCanCleanFolderAsync(scopeInfo.Name, context.Parameters, batchInfo, cancellationToken).ConfigureAwait(false); batchInfo.Clear(cleanFolder); } var databaseChangesSelectedArgs = new DatabaseChangesSelectedArgs(context, fromLastTimestamp, toNewTimestamp, batchInfo, changesSelected, connection); await this.InterceptAsync(databaseChangesSelectedArgs, progress, cancellationToken).ConfigureAwait(false); return(context, batchInfo, changesSelected); }