/// <summary> /// Create a response message content based on a requested index in a server batch info /// </summary> private HttpMessageSendChangesResponse GetChangesResponse(SyncContext syncContext, long remoteClientTimestamp, BatchInfo serverBatchInfo, DatabaseChangesSelected serverChangesSelected, int batchIndexRequested, ConflictResolutionPolicy policy) { // 1) Create the http message content response var changesResponse = new HttpMessageSendChangesResponse(syncContext); changesResponse.ChangesSelected = serverChangesSelected; changesResponse.ServerStep = HttpStep.GetChanges; changesResponse.ConflictResolutionPolicy = policy; // If nothing to do, just send back if (serverBatchInfo.InMemory || serverBatchInfo.BatchPartsInfo.Count == 0) { if (this.ClientConverter != null && serverBatchInfo.InMemoryData.HasRows) { BeforeSerializeRows(serverBatchInfo.InMemoryData, this.ClientConverter); } changesResponse.Changes = serverBatchInfo.InMemoryData.GetContainerSet(); changesResponse.BatchIndex = 0; changesResponse.IsLastBatch = true; changesResponse.RemoteClientTimestamp = remoteClientTimestamp; return(changesResponse); } // Get the batch part index requested var batchPartInfo = serverBatchInfo.BatchPartsInfo.First(d => d.Index == batchIndexRequested); // if we are not in memory, we set the BI in session, to be able to get it back on next request // create the in memory changes set var changesSet = new SyncSet(Schema.ScopeName); foreach (var table in Schema.Tables) { DbSyncAdapter.CreateChangesTable(Schema.Tables[table.TableName, table.SchemaName], changesSet); } batchPartInfo.LoadBatch(changesSet, serverBatchInfo.GetDirectoryFullPath()); // if client request a conversion on each row, apply the conversion if (this.ClientConverter != null && batchPartInfo.Data.HasRows) { BeforeSerializeRows(batchPartInfo.Data, this.ClientConverter); } changesResponse.Changes = batchPartInfo.Data.GetContainerSet(); changesResponse.BatchIndex = batchIndexRequested; changesResponse.IsLastBatch = batchPartInfo.IsLastBatch; changesResponse.RemoteClientTimestamp = remoteClientTimestamp; changesResponse.ServerStep = batchPartInfo.IsLastBatch ? HttpStep.GetChanges : HttpStep.GetChangesInProgress; // If we have only one bpi, we can safely delete it if (batchPartInfo.IsLastBatch) { // delete the folder (not the BatchPartInfo, because we have a reference on it) if (this.Options.CleanFolder) { var shouldDeleteFolder = true; if (!string.IsNullOrEmpty(this.Options.SnapshotsDirectory)) { var dirInfo = new DirectoryInfo(serverBatchInfo.DirectoryRoot); var snapInfo = new DirectoryInfo(this.Options.SnapshotsDirectory); shouldDeleteFolder = dirInfo.FullName != snapInfo.FullName; } if (shouldDeleteFolder) { serverBatchInfo.TryRemoveDirectory(); } } } return(changesResponse); }
/// <summary> /// Gets a batch of changes to synchronize when given batch size, /// destination knowledge, and change data retriever parameters. /// </summary> /// <returns>A DbSyncContext object that will be used to retrieve the modified data.</returns> internal virtual async Task <(SyncContext, BatchInfo, DatabaseChangesSelected)> InternalGetChangesAsync( IScopeInfo scopeInfo, SyncContext context, bool isNew, long?fromLastTimestamp, long?toNewTimestamp, Guid?excludingScopeId, bool supportsMultiActiveResultSets, string batchRootDirectory, string batchDirectoryName, DbConnection connection, DbTransaction transaction, CancellationToken cancellationToken, IProgress <ProgressArgs> progress) { // batch info containing changes BatchInfo batchInfo; // Statistics about changes that are selected DatabaseChangesSelected changesSelected; context.SyncStage = SyncStage.ChangesSelecting; if (context.SyncWay == SyncWay.Upload && context.SyncType == SyncType.Reinitialize) { (batchInfo, changesSelected) = await this.InternalGetEmptyChangesAsync(scopeInfo, batchRootDirectory).ConfigureAwait(false); return(context, batchInfo, changesSelected); } // create local directory if (!string.IsNullOrEmpty(batchRootDirectory) && !Directory.Exists(batchRootDirectory)) { Directory.CreateDirectory(batchRootDirectory); } changesSelected = new DatabaseChangesSelected(); // Create a batch // batchinfo generate a schema clone with scope columns if needed batchInfo = new BatchInfo(scopeInfo.Schema, batchRootDirectory, batchDirectoryName); batchInfo.TryRemoveDirectory(); batchInfo.CreateDirectory(); // Call interceptor var databaseChangesSelectingArgs = new DatabaseChangesSelectingArgs(context, batchInfo.GetDirectoryFullPath(), this.Options.BatchSize, isNew, fromLastTimestamp, toNewTimestamp, connection, transaction); await this.InterceptAsync(databaseChangesSelectingArgs, progress, cancellationToken).ConfigureAwait(false); var cptSyncTable = 0; var currentProgress = context.ProgressPercentage; var schemaTables = scopeInfo.Schema.Tables.SortByDependencies(tab => tab.GetRelations().Select(r => r.GetParentTable())); var lstAllBatchPartInfos = new ConcurrentBag <BatchPartInfo>(); var lstTableChangesSelected = new ConcurrentBag <TableChangesSelected>(); var threadNumberLimits = supportsMultiActiveResultSets ? 16 : 1; if (supportsMultiActiveResultSets) { await schemaTables.ForEachAsync(async syncTable => { if (cancellationToken.IsCancellationRequested) { return; } // tmp count of table for report progress pct cptSyncTable++; List <BatchPartInfo> syncTableBatchPartInfos; TableChangesSelected tableChangesSelected; (context, syncTableBatchPartInfos, tableChangesSelected) = await InternalReadSyncTableChangesAsync( scopeInfo, context, excludingScopeId, syncTable, batchInfo, isNew, fromLastTimestamp, connection, transaction, cancellationToken, progress).ConfigureAwait(false); if (syncTableBatchPartInfos == null) { return; } // We don't report progress if no table changes is empty, to limit verbosity if (tableChangesSelected != null && (tableChangesSelected.Deletes > 0 || tableChangesSelected.Upserts > 0)) { lstTableChangesSelected.Add(tableChangesSelected); } // Add sync table bpi to all bpi syncTableBatchPartInfos.ForEach(bpi => lstAllBatchPartInfos.Add(bpi)); context.ProgressPercentage = currentProgress + (cptSyncTable * 0.2d / scopeInfo.Schema.Tables.Count); }, threadNumberLimits); } else { foreach (var syncTable in schemaTables) { if (cancellationToken.IsCancellationRequested) { continue; } // tmp count of table for report progress pct cptSyncTable++; List <BatchPartInfo> syncTableBatchPartInfos; TableChangesSelected tableChangesSelected; (context, syncTableBatchPartInfos, tableChangesSelected) = await InternalReadSyncTableChangesAsync( scopeInfo, context, excludingScopeId, syncTable, batchInfo, isNew, fromLastTimestamp, connection, transaction, cancellationToken, progress).ConfigureAwait(false); if (syncTableBatchPartInfos == null) { continue; } // We don't report progress if no table changes is empty, to limit verbosity if (tableChangesSelected != null && (tableChangesSelected.Deletes > 0 || tableChangesSelected.Upserts > 0)) { lstTableChangesSelected.Add(tableChangesSelected); } // Add sync table bpi to all bpi syncTableBatchPartInfos.ForEach(bpi => lstAllBatchPartInfos.Add(bpi)); context.ProgressPercentage = currentProgress + (cptSyncTable * 0.2d / scopeInfo.Schema.Tables.Count); } } while (!lstTableChangesSelected.IsEmpty) { if (lstTableChangesSelected.TryTake(out var tableChangesSelected)) { changesSelected.TableChangesSelected.Add(tableChangesSelected); } } // delete all empty batchparts (empty tables) foreach (var bpi in lstAllBatchPartInfos.Where(bpi => bpi.RowsCount <= 0)) { File.Delete(Path.Combine(batchInfo.GetDirectoryFullPath(), bpi.FileName)); } // Generate a good index order to be compliant with previous versions var tmpLstBatchPartInfos = new List <BatchPartInfo>(); foreach (var table in schemaTables) { // get all bpi where count > 0 and ordered by index foreach (var bpi in lstAllBatchPartInfos.Where(bpi => bpi.RowsCount > 0 && bpi.Tables[0].EqualsByName(new BatchPartTableInfo(table.TableName, table.SchemaName))).OrderBy(bpi => bpi.Index).ToArray()) { batchInfo.BatchPartsInfo.Add(bpi); batchInfo.RowsCount += bpi.RowsCount; tmpLstBatchPartInfos.Add(bpi); } } var newBatchIndex = 0; foreach (var bpi in tmpLstBatchPartInfos) { bpi.Index = newBatchIndex; newBatchIndex++; bpi.IsLastBatch = newBatchIndex == tmpLstBatchPartInfos.Count; } //Set the total rows count contained in the batch info batchInfo.EnsureLastBatch(); if (batchInfo.RowsCount <= 0) { var cleanFolder = await this.InternalCanCleanFolderAsync(scopeInfo.Name, context.Parameters, batchInfo, cancellationToken).ConfigureAwait(false); batchInfo.Clear(cleanFolder); } var databaseChangesSelectedArgs = new DatabaseChangesSelectedArgs(context, fromLastTimestamp, toNewTimestamp, batchInfo, changesSelected, connection); await this.InterceptAsync(databaseChangesSelectedArgs, progress, cancellationToken).ConfigureAwait(false); return(context, batchInfo, changesSelected); }