/// <summary> /// Gets a batch of changes to synchronize when given batch size, /// destination knowledge, and change data retriever parameters. /// </summary> /// <returns>A DbSyncContext object that will be used to retrieve the modified data.</returns> internal virtual async Task <(SyncContext, BatchInfo, DatabaseChangesSelected)> InternalGetChangesAsync( IScopeInfo scopeInfo, SyncContext context, bool isNew, long?fromLastTimestamp, long?toNewTimestamp, Guid?excludingScopeId, bool supportsMultiActiveResultSets, string batchRootDirectory, string batchDirectoryName, DbConnection connection, DbTransaction transaction, CancellationToken cancellationToken, IProgress <ProgressArgs> progress) { // batch info containing changes BatchInfo batchInfo; // Statistics about changes that are selected DatabaseChangesSelected changesSelected; context.SyncStage = SyncStage.ChangesSelecting; if (context.SyncWay == SyncWay.Upload && context.SyncType == SyncType.Reinitialize) { (batchInfo, changesSelected) = await this.InternalGetEmptyChangesAsync(scopeInfo, batchRootDirectory).ConfigureAwait(false); return(context, batchInfo, changesSelected); } // create local directory if (!string.IsNullOrEmpty(batchRootDirectory) && !Directory.Exists(batchRootDirectory)) { Directory.CreateDirectory(batchRootDirectory); } changesSelected = new DatabaseChangesSelected(); // Create a batch // batchinfo generate a schema clone with scope columns if needed batchInfo = new BatchInfo(scopeInfo.Schema, batchRootDirectory, batchDirectoryName); batchInfo.TryRemoveDirectory(); batchInfo.CreateDirectory(); // Call interceptor var databaseChangesSelectingArgs = new DatabaseChangesSelectingArgs(context, batchInfo.GetDirectoryFullPath(), this.Options.BatchSize, isNew, fromLastTimestamp, toNewTimestamp, connection, transaction); await this.InterceptAsync(databaseChangesSelectingArgs, progress, cancellationToken).ConfigureAwait(false); var cptSyncTable = 0; var currentProgress = context.ProgressPercentage; var schemaTables = scopeInfo.Schema.Tables.SortByDependencies(tab => tab.GetRelations().Select(r => r.GetParentTable())); var lstAllBatchPartInfos = new ConcurrentBag <BatchPartInfo>(); var lstTableChangesSelected = new ConcurrentBag <TableChangesSelected>(); var threadNumberLimits = supportsMultiActiveResultSets ? 16 : 1; if (supportsMultiActiveResultSets) { await schemaTables.ForEachAsync(async syncTable => { if (cancellationToken.IsCancellationRequested) { return; } // tmp count of table for report progress pct cptSyncTable++; List <BatchPartInfo> syncTableBatchPartInfos; TableChangesSelected tableChangesSelected; (context, syncTableBatchPartInfos, tableChangesSelected) = await InternalReadSyncTableChangesAsync( scopeInfo, context, excludingScopeId, syncTable, batchInfo, isNew, fromLastTimestamp, connection, transaction, cancellationToken, progress).ConfigureAwait(false); if (syncTableBatchPartInfos == null) { return; } // We don't report progress if no table changes is empty, to limit verbosity if (tableChangesSelected != null && (tableChangesSelected.Deletes > 0 || tableChangesSelected.Upserts > 0)) { lstTableChangesSelected.Add(tableChangesSelected); } // Add sync table bpi to all bpi syncTableBatchPartInfos.ForEach(bpi => lstAllBatchPartInfos.Add(bpi)); context.ProgressPercentage = currentProgress + (cptSyncTable * 0.2d / scopeInfo.Schema.Tables.Count); }, threadNumberLimits); } else { foreach (var syncTable in schemaTables) { if (cancellationToken.IsCancellationRequested) { continue; } // tmp count of table for report progress pct cptSyncTable++; List <BatchPartInfo> syncTableBatchPartInfos; TableChangesSelected tableChangesSelected; (context, syncTableBatchPartInfos, tableChangesSelected) = await InternalReadSyncTableChangesAsync( scopeInfo, context, excludingScopeId, syncTable, batchInfo, isNew, fromLastTimestamp, connection, transaction, cancellationToken, progress).ConfigureAwait(false); if (syncTableBatchPartInfos == null) { continue; } // We don't report progress if no table changes is empty, to limit verbosity if (tableChangesSelected != null && (tableChangesSelected.Deletes > 0 || tableChangesSelected.Upserts > 0)) { lstTableChangesSelected.Add(tableChangesSelected); } // Add sync table bpi to all bpi syncTableBatchPartInfos.ForEach(bpi => lstAllBatchPartInfos.Add(bpi)); context.ProgressPercentage = currentProgress + (cptSyncTable * 0.2d / scopeInfo.Schema.Tables.Count); } } while (!lstTableChangesSelected.IsEmpty) { if (lstTableChangesSelected.TryTake(out var tableChangesSelected)) { changesSelected.TableChangesSelected.Add(tableChangesSelected); } } // delete all empty batchparts (empty tables) foreach (var bpi in lstAllBatchPartInfos.Where(bpi => bpi.RowsCount <= 0)) { File.Delete(Path.Combine(batchInfo.GetDirectoryFullPath(), bpi.FileName)); } // Generate a good index order to be compliant with previous versions var tmpLstBatchPartInfos = new List <BatchPartInfo>(); foreach (var table in schemaTables) { // get all bpi where count > 0 and ordered by index foreach (var bpi in lstAllBatchPartInfos.Where(bpi => bpi.RowsCount > 0 && bpi.Tables[0].EqualsByName(new BatchPartTableInfo(table.TableName, table.SchemaName))).OrderBy(bpi => bpi.Index).ToArray()) { batchInfo.BatchPartsInfo.Add(bpi); batchInfo.RowsCount += bpi.RowsCount; tmpLstBatchPartInfos.Add(bpi); } } var newBatchIndex = 0; foreach (var bpi in tmpLstBatchPartInfos) { bpi.Index = newBatchIndex; newBatchIndex++; bpi.IsLastBatch = newBatchIndex == tmpLstBatchPartInfos.Count; } //Set the total rows count contained in the batch info batchInfo.EnsureLastBatch(); if (batchInfo.RowsCount <= 0) { var cleanFolder = await this.InternalCanCleanFolderAsync(scopeInfo.Name, context.Parameters, batchInfo, cancellationToken).ConfigureAwait(false); batchInfo.Clear(cleanFolder); } var databaseChangesSelectedArgs = new DatabaseChangesSelectedArgs(context, fromLastTimestamp, toNewTimestamp, batchInfo, changesSelected, connection); await this.InterceptAsync(databaseChangesSelectedArgs, progress, cancellationToken).ConfigureAwait(false); return(context, batchInfo, changesSelected); }