InternalApplyThenGetChangesAsync(ClientScopeInfo clientScopeInfo, SyncContext context, BatchInfo clientBatchInfo, DbConnection connection = default, DbTransaction transaction = default, CancellationToken cancellationToken = default, IProgress <ProgressArgs> progress = null)
{
await using var runner = await this.GetConnectionAsync(context, SyncMode.Reading, SyncStage.ChangesApplying, connection, transaction, cancellationToken, progress).ConfigureAwait(false);
SyncSet schema;
ServerScopeInfo serverScopeInfo;
// is it something that could happens ?
if (clientScopeInfo.Schema == null)
{
// Make a remote call to get Schema from remote provider
(context, serverScopeInfo) = await this.InternalGetServerScopeInfoAsync(
context, null, runner.Connection, runner.Transaction, runner.CancellationToken, runner.Progress).ConfigureAwait(false);
schema = serverScopeInfo.Schema;
}
else
{
schema = clientScopeInfo.Schema;
}
schema.EnsureSchema();
// if we don't have any BatchPartsInfo, just generate a new one to get, at least, something to send to the server
// and get a response with new data from server
if (clientBatchInfo == null)
{
clientBatchInfo = new BatchInfo(schema);
}
// --------------------------------------------------------------
// STEP 1 : Send everything to the server side
// --------------------------------------------------------------
HttpResponseMessage response = null;
// If not in memory and BatchPartsInfo.Count == 0, nothing to send.
// But we need to send something, so generate a little batch part
if (clientBatchInfo.BatchPartsInfo.Count == 0)
{
var changesToSend = new HttpMessageSendChangesRequest(context, clientScopeInfo);
var containerSet = new ContainerSet();
changesToSend.Changes = containerSet;
changesToSend.IsLastBatch = true;
changesToSend.BatchIndex = 0;
changesToSend.BatchCount = clientBatchInfo.BatchPartsInfo == null ? 0 : clientBatchInfo.BatchPartsInfo.Count;
var inMemoryRowsCount = changesToSend.Changes.RowsCount();
context.ProgressPercentage += 0.125;
await this.InterceptAsync(new HttpSendingClientChangesRequestArgs(changesToSend, inMemoryRowsCount, inMemoryRowsCount, this.GetServiceHost()), progress, cancellationToken).ConfigureAwait(false);
// serialize message
var serializer = this.SerializerFactory.GetSerializer <HttpMessageSendChangesRequest>();
var binaryData = await serializer.SerializeAsync(changesToSend);
response = await this.httpRequestHandler.ProcessRequestAsync
(this.HttpClient, context, this.ServiceUri, binaryData, HttpStep.SendChangesInProgress,
this.SerializerFactory, this.Converter, this.Options.BatchSize, this.SyncPolicy, cancellationToken, progress).ConfigureAwait(false);
}
else
{
int tmpRowsSendedCount = 0;
// Foreach part, will have to send them to the remote
// once finished, return context
var initialPctProgress1 = context.ProgressPercentage;
var localSerializer = new LocalJsonSerializer();
var interceptorsReading = this.interceptors.GetInterceptors <DeserializingRowArgs>();
if (interceptorsReading.Count > 0)
{
localSerializer.OnReadingRow(async(schemaTable, rowString) =>
{
var args = new DeserializingRowArgs(context, schemaTable, rowString);
await this.InterceptAsync(args);
return(args.Result);
});
}
foreach (var bpi in clientBatchInfo.BatchPartsInfo.OrderBy(bpi => bpi.Index))
{
// Get the updatable schema for the only table contained in the batchpartinfo
var schemaTable = DbSyncAdapter.CreateChangesTable(schema.Tables[bpi.Tables[0].TableName, bpi.Tables[0].SchemaName]);
// Generate the ContainerSet containing rows to send to the user
var containerSet = new ContainerSet();
var containerTable = new ContainerTable(schemaTable);
var fullPath = Path.Combine(clientBatchInfo.GetDirectoryFullPath(), bpi.FileName);
containerSet.Tables.Add(containerTable);
// read rows from file
foreach (var row in localSerializer.ReadRowsFromFile(fullPath, schemaTable))
{
containerTable.Rows.Add(row.ToArray());
}
// Call the converter if needed
if (this.Converter != null && containerTable.HasRows)
{
BeforeSerializeRows(containerTable, schemaTable, this.Converter);
}
// Create the send changes request
var changesToSend = new HttpMessageSendChangesRequest(context, clientScopeInfo)
{
Changes = containerSet,
IsLastBatch = bpi.IsLastBatch,
BatchIndex = bpi.Index,
BatchCount = clientBatchInfo.BatchPartsInfo.Count
};
tmpRowsSendedCount += containerTable.Rows.Count;
context.ProgressPercentage = initialPctProgress1 + ((changesToSend.BatchIndex + 1) * 0.2d / changesToSend.BatchCount);
await this.InterceptAsync(new HttpSendingClientChangesRequestArgs(changesToSend, tmpRowsSendedCount, clientBatchInfo.RowsCount, this.GetServiceHost()), progress, cancellationToken).ConfigureAwait(false);
// serialize message
var serializer = this.SerializerFactory.GetSerializer <HttpMessageSendChangesRequest>();
var binaryData = await serializer.SerializeAsync(changesToSend);
response = await this.httpRequestHandler.ProcessRequestAsync
(this.HttpClient, context, this.ServiceUri, binaryData, HttpStep.SendChangesInProgress,
this.SerializerFactory, this.Converter, this.Options.BatchSize, this.SyncPolicy, cancellationToken, progress).ConfigureAwait(false);
// See #721 for issue and #721 for PR from slagtejn
if (!bpi.IsLastBatch)
{
response.Dispose();
}
}
}
// --------------------------------------------------------------
// STEP 2 : Receive everything from the server side
// --------------------------------------------------------------
// Now we have sent all the datas to the server and now :
// We have a FIRST response from the server with new datas
// 1) Could be the only one response
// 2) Could be the first response and we need to download all batchs
context.SyncStage = SyncStage.ChangesSelecting;
var initialPctProgress = 0.55;
context.ProgressPercentage = initialPctProgress;
// Create the BatchInfo
var serverBatchInfo = new BatchInfo(schema);
HttpMessageSummaryResponse summaryResponseContent = null;
// Deserialize response incoming from server
using (var streamResponse = await response.Content.ReadAsStreamAsync().ConfigureAwait(false))
{
var responseSerializer = this.SerializerFactory.GetSerializer <HttpMessageSummaryResponse>();
summaryResponseContent = await responseSerializer.DeserializeAsync(streamResponse);
}
serverBatchInfo.RowsCount = summaryResponseContent.BatchInfo.RowsCount;
serverBatchInfo.Timestamp = summaryResponseContent.RemoteClientTimestamp;
context = summaryResponseContent.SyncContext;
if (summaryResponseContent.BatchInfo.BatchPartsInfo != null)
{
foreach (var bpi in summaryResponseContent.BatchInfo.BatchPartsInfo)
{
serverBatchInfo.BatchPartsInfo.Add(bpi);
}
}
// From here, we need to serialize everything on disk
// Generate the batch directory
var batchDirectoryRoot = this.Options.BatchDirectory;
var batchDirectoryName = string.Concat(DateTime.UtcNow.ToString("yyyy_MM_dd_ss"), Path.GetRandomFileName().Replace(".", ""));
serverBatchInfo.DirectoryRoot = batchDirectoryRoot;
serverBatchInfo.DirectoryName = batchDirectoryName;
if (!Directory.Exists(serverBatchInfo.GetDirectoryFullPath()))
{
Directory.CreateDirectory(serverBatchInfo.GetDirectoryFullPath());
}
// If we have a snapshot we are raising the batches downloading process that will occurs
await this.InterceptAsync(new HttpBatchesDownloadingArgs(context, serverBatchInfo, this.GetServiceHost()), progress, cancellationToken).ConfigureAwait(false);
// function used to download one part
var dl = new Func <BatchPartInfo, Task>(async(bpi) =>
{
if (cancellationToken.IsCancellationRequested)
{
return;
}
var changesToSend3 = new HttpMessageGetMoreChangesRequest(context, bpi.Index);
var serializer3 = this.SerializerFactory.GetSerializer <HttpMessageGetMoreChangesRequest>();
var binaryData3 = await serializer3.SerializeAsync(changesToSend3).ConfigureAwait(false);
var step3 = HttpStep.GetMoreChanges;
await this.InterceptAsync(new HttpGettingServerChangesRequestArgs(bpi.Index, serverBatchInfo.BatchPartsInfo.Count, summaryResponseContent.SyncContext, this.GetServiceHost()), progress, cancellationToken).ConfigureAwait(false);
// Raise get changes request
context.ProgressPercentage = initialPctProgress + ((bpi.Index + 1) * 0.2d / serverBatchInfo.BatchPartsInfo.Count);
var response = await this.httpRequestHandler.ProcessRequestAsync(
this.HttpClient, context, this.ServiceUri, binaryData3, step3,
this.SerializerFactory, this.Converter, 0, this.SyncPolicy, cancellationToken, progress).ConfigureAwait(false);
if (this.SerializerFactory.Key != "json")
{
var webSerializer = this.SerializerFactory.GetSerializer <HttpMessageSendChangesResponse>();
using var responseStream = await response.Content.ReadAsStreamAsync().ConfigureAwait(false);
var getMoreChanges = await webSerializer.DeserializeAsync(responseStream);
context = getMoreChanges.SyncContext;
if (getMoreChanges != null && getMoreChanges.Changes != null && getMoreChanges.Changes.HasRows)
{
var localSerializer = new LocalJsonSerializer();
var interceptorsWriting = this.interceptors.GetInterceptors <SerializingRowArgs>();
if (interceptorsWriting.Count > 0)
{
localSerializer.OnWritingRow(async(syncTable, rowArray) =>
{
var args = new SerializingRowArgs(context, syncTable, rowArray);
await this.InterceptAsync(args, progress, cancellationToken).ConfigureAwait(false);
return(args.Result);
});
}
// Should have only one table
var table = getMoreChanges.Changes.Tables[0];
var schemaTable = DbSyncAdapter.CreateChangesTable(schema.Tables[table.TableName, table.SchemaName]);
var fullPath = Path.Combine(serverBatchInfo.GetDirectoryFullPath(), bpi.FileName);
// open the file and write table header
await localSerializer.OpenFileAsync(fullPath, schemaTable).ConfigureAwait(false);
foreach (var row in table.Rows)
{
await localSerializer.WriteRowToFileAsync(new SyncRow(schemaTable, row), schemaTable).ConfigureAwait(false);
}
// Close file
await localSerializer.CloseFileAsync(fullPath, schemaTable).ConfigureAwait(false);
}
}
else
{
// Serialize
await SerializeAsync(response, bpi.FileName, serverBatchInfo.GetDirectoryFullPath(), this).ConfigureAwait(false);
}
// Raise response from server containing a batch changes
await this.InterceptAsync(new HttpGettingServerChangesResponseArgs(serverBatchInfo, bpi.Index, bpi.RowsCount, summaryResponseContent.SyncContext, this.GetServiceHost()), progress, cancellationToken).ConfigureAwait(false);
});
// Parrallel download of all bpis (which will launch the delete directory on the server side)
await serverBatchInfo.BatchPartsInfo.ForEachAsync(bpi => dl(bpi), this.MaxDownladingDegreeOfParallelism).ConfigureAwait(false);
// Send order of end of download
var lastBpi = serverBatchInfo.BatchPartsInfo.FirstOrDefault(bpi => bpi.IsLastBatch);
if (lastBpi != null)
{
var endOfDownloadChanges = new HttpMessageGetMoreChangesRequest(context, lastBpi.Index);
var serializerEndOfDownloadChanges = this.SerializerFactory.GetSerializer <HttpMessageGetMoreChangesRequest>();
var binaryData3 = await serializerEndOfDownloadChanges.SerializeAsync(endOfDownloadChanges).ConfigureAwait(false);
var endResponse = await this.httpRequestHandler.ProcessRequestAsync(
this.HttpClient, context, this.ServiceUri, binaryData3, HttpStep.SendEndDownloadChanges,
this.SerializerFactory, this.Converter, 0, this.SyncPolicy, cancellationToken, progress).ConfigureAwait(false);
// Deserialize response incoming from server
// This is the last response
// Should contains step HttpStep.SendEndDownloadChanges
using var streamResponse = await endResponse.Content.ReadAsStreamAsync().ConfigureAwait(false);
var endResponseSerializer = this.SerializerFactory.GetSerializer <HttpMessageSendChangesResponse>();
var endResponseContent = await endResponseSerializer.DeserializeAsync(streamResponse);
context = endResponseContent.SyncContext;
}
// generate the new scope item
this.CompleteTime = DateTime.UtcNow;
await this.InterceptAsync(new HttpBatchesDownloadedArgs(summaryResponseContent, summaryResponseContent.SyncContext, this.GetServiceHost()), progress, cancellationToken).ConfigureAwait(false);
var serverSyncChanges = new ServerSyncChanges(
summaryResponseContent.RemoteClientTimestamp,
serverBatchInfo,
summaryResponseContent.ServerChangesSelected
);
return(context, serverSyncChanges, summaryResponseContent.ClientChangesApplied, summaryResponseContent.ConflictResolutionPolicy);
}
/// <summary>
/// Create a response message content based on a requested index in a server batch info
/// </summary>
private HttpMessageSendChangesResponse GetChangesResponse(SyncContext syncContext, long remoteClientTimestamp, BatchInfo serverBatchInfo,
DatabaseChangesSelected serverChangesSelected, int batchIndexRequested, ConflictResolutionPolicy policy)
{
// 1) Create the http message content response
var changesResponse = new HttpMessageSendChangesResponse(syncContext);
changesResponse.ChangesSelected = serverChangesSelected;
changesResponse.ServerStep = HttpStep.GetChanges;
changesResponse.ConflictResolutionPolicy = policy;
// If nothing to do, just send back
if (serverBatchInfo.InMemory || serverBatchInfo.BatchPartsInfo.Count == 0)
{
if (this.ClientConverter != null && serverBatchInfo.InMemoryData.HasRows)
{
BeforeSerializeRows(serverBatchInfo.InMemoryData, this.ClientConverter);
}
changesResponse.Changes = serverBatchInfo.InMemoryData.GetContainerSet();
changesResponse.BatchIndex = 0;
changesResponse.IsLastBatch = true;
changesResponse.RemoteClientTimestamp = remoteClientTimestamp;
return(changesResponse);
}
// Get the batch part index requested
var batchPartInfo = serverBatchInfo.BatchPartsInfo.First(d => d.Index == batchIndexRequested);
// if we are not in memory, we set the BI in session, to be able to get it back on next request
// create the in memory changes set
var changesSet = new SyncSet(Schema.ScopeName);
foreach (var table in Schema.Tables)
{
DbSyncAdapter.CreateChangesTable(Schema.Tables[table.TableName, table.SchemaName], changesSet);
}
batchPartInfo.LoadBatch(changesSet, serverBatchInfo.GetDirectoryFullPath());
// if client request a conversion on each row, apply the conversion
if (this.ClientConverter != null && batchPartInfo.Data.HasRows)
{
BeforeSerializeRows(batchPartInfo.Data, this.ClientConverter);
}
changesResponse.Changes = batchPartInfo.Data.GetContainerSet();
changesResponse.BatchIndex = batchIndexRequested;
changesResponse.IsLastBatch = batchPartInfo.IsLastBatch;
changesResponse.RemoteClientTimestamp = remoteClientTimestamp;
changesResponse.ServerStep = batchPartInfo.IsLastBatch ? HttpStep.GetChanges : HttpStep.GetChangesInProgress;
// If we have only one bpi, we can safely delete it
if (batchPartInfo.IsLastBatch)
{
// delete the folder (not the BatchPartInfo, because we have a reference on it)
if (this.Options.CleanFolder)
{
var shouldDeleteFolder = true;
if (!string.IsNullOrEmpty(this.Options.SnapshotsDirectory))
{
var dirInfo = new DirectoryInfo(serverBatchInfo.DirectoryRoot);
var snapInfo = new DirectoryInfo(this.Options.SnapshotsDirectory);
shouldDeleteFolder = dirInfo.FullName != snapInfo.FullName;
}
if (shouldDeleteFolder)
{
serverBatchInfo.TryRemoveDirectory();
}
}
}
return(changesResponse);
}
GetChangesAsync(ClientScopeInfo clientScopeInfo, SyncParameters parameters = default, DbConnection connection = default, DbTransaction transaction = default, CancellationToken cancellationToken = default, IProgress <ProgressArgs> progress = null)
{
var context = new SyncContext(Guid.NewGuid(), clientScopeInfo.Name);
if (parameters != null)
{
context.Parameters = parameters;
}
SyncSet schema;
ServerScopeInfo serverScopeInfo;
// Need the server scope
(context, serverScopeInfo) = await this.InternalGetServerScopeInfoAsync(context, clientScopeInfo.Setup, connection, transaction, cancellationToken, progress).ConfigureAwait(false);
schema = serverScopeInfo.Schema;
schema.EnsureSchema();
clientScopeInfo.Schema = schema;
clientScopeInfo.Setup = serverScopeInfo.Setup;
clientScopeInfo.Version = serverScopeInfo.Version;
var changesToSend = new HttpMessageSendChangesRequest(context, clientScopeInfo);
var containerSet = new ContainerSet();
changesToSend.Changes = containerSet;
changesToSend.IsLastBatch = true;
changesToSend.BatchIndex = 0;
changesToSend.BatchCount = 0;
context.ProgressPercentage += 0.125;
await this.InterceptAsync(new HttpSendingClientChangesRequestArgs(changesToSend, 0, 0, this.GetServiceHost()), progress, cancellationToken).ConfigureAwait(false);
// serialize message
var serializer = this.SerializerFactory.GetSerializer <HttpMessageSendChangesRequest>();
var binaryData = await serializer.SerializeAsync(changesToSend);
var response = await this.httpRequestHandler.ProcessRequestAsync
(this.HttpClient, context, this.ServiceUri, binaryData, HttpStep.SendChangesInProgress,
this.SerializerFactory, this.Converter, this.Options.BatchSize, this.SyncPolicy, cancellationToken, progress).ConfigureAwait(false);
// --------------------------------------------------------------
// STEP 2 : Receive everything from the server side
// --------------------------------------------------------------
// Now we have sent all the datas to the server and now :
// We have a FIRST response from the server with new datas
// 1) Could be the only one response (enough or InMemory is set on the server side)
// 2) Could bt the first response and we need to download all batchs
context.SyncStage = SyncStage.ChangesSelecting;
var initialPctProgress = 0.55;
context.ProgressPercentage = initialPctProgress;
// Create the BatchInfo
var serverBatchInfo = new BatchInfo(schema);
HttpMessageSummaryResponse summaryResponseContent = null;
// Deserialize response incoming from server
using (var streamResponse = await response.Content.ReadAsStreamAsync().ConfigureAwait(false))
{
var responseSerializer = this.SerializerFactory.GetSerializer <HttpMessageSummaryResponse>();
summaryResponseContent = await responseSerializer.DeserializeAsync(streamResponse);
}
serverBatchInfo.RowsCount = summaryResponseContent.BatchInfo.RowsCount;
serverBatchInfo.Timestamp = summaryResponseContent.RemoteClientTimestamp;
context = summaryResponseContent.SyncContext;
if (summaryResponseContent.BatchInfo.BatchPartsInfo != null)
{
foreach (var bpi in summaryResponseContent.BatchInfo.BatchPartsInfo)
{
serverBatchInfo.BatchPartsInfo.Add(bpi);
}
}
//-----------------------
// In Batch Mode
//-----------------------
// From here, we need to serialize everything on disk
// Generate the batch directory
var batchDirectoryRoot = this.Options.BatchDirectory;
var batchDirectoryName = string.Concat(DateTime.UtcNow.ToString("yyyy_MM_dd_ss"), Path.GetRandomFileName().Replace(".", ""));
serverBatchInfo.DirectoryRoot = batchDirectoryRoot;
serverBatchInfo.DirectoryName = batchDirectoryName;
if (!Directory.Exists(serverBatchInfo.GetDirectoryFullPath()))
{
Directory.CreateDirectory(serverBatchInfo.GetDirectoryFullPath());
}
// hook to get the last batch part info at the end
var bpis = serverBatchInfo.BatchPartsInfo.Where(bpi => !bpi.IsLastBatch);
var lstbpi = serverBatchInfo.BatchPartsInfo.First(bpi => bpi.IsLastBatch);
// function used to download one part
var dl = new Func <BatchPartInfo, Task>(async(bpi) =>
{
var changesToSend3 = new HttpMessageGetMoreChangesRequest(context, bpi.Index);
var serializer3 = this.SerializerFactory.GetSerializer <HttpMessageGetMoreChangesRequest>();
var binaryData3 = await serializer3.SerializeAsync(changesToSend3).ConfigureAwait(false);
var step3 = HttpStep.GetMoreChanges;
await this.InterceptAsync(new HttpGettingServerChangesRequestArgs(bpi.Index, serverBatchInfo.BatchPartsInfo.Count, summaryResponseContent.SyncContext, this.GetServiceHost()), progress, cancellationToken).ConfigureAwait(false);
// Raise get changes request
context.ProgressPercentage = initialPctProgress + ((bpi.Index + 1) * 0.2d / serverBatchInfo.BatchPartsInfo.Count);
var response = await this.httpRequestHandler.ProcessRequestAsync(
this.HttpClient, context, this.ServiceUri, binaryData3, step3,
this.SerializerFactory, this.Converter, 0, this.SyncPolicy, cancellationToken, progress).ConfigureAwait(false);
// Serialize
await SerializeAsync(response, bpi.FileName, serverBatchInfo.GetDirectoryFullPath(), this).ConfigureAwait(false);
// Raise response from server containing a batch changes
await this.InterceptAsync(new HttpGettingServerChangesResponseArgs(serverBatchInfo, bpi.Index, bpi.RowsCount, summaryResponseContent.SyncContext, this.GetServiceHost()), progress, cancellationToken).ConfigureAwait(false);
response.Dispose();
});
// Parrallel download of all bpis except the last one (which will launch the delete directory on the server side)
await bpis.ForEachAsync(bpi => dl(bpi), this.MaxDownladingDegreeOfParallelism).ConfigureAwait(false);
// Download last batch part that will launch the server deletion of the tmp dir
await dl(lstbpi).ConfigureAwait(false);
// generate the new scope item
this.CompleteTime = DateTime.UtcNow;
// Reaffect context
context = summaryResponseContent.SyncContext;
return(new ServerSyncChanges(summaryResponseContent.RemoteClientTimestamp, serverBatchInfo, summaryResponseContent.ServerChangesSelected));
}
/// <summary>
/// Gets a batch of changes to synchronize when given batch size,
/// destination knowledge, and change data retriever parameters.
/// </summary>
/// <returns>A DbSyncContext object that will be used to retrieve the modified data.</returns>
internal virtual async Task <(SyncContext, BatchInfo, DatabaseChangesSelected)> InternalGetChangesAsync(
IScopeInfo scopeInfo, SyncContext context, bool isNew, long?fromLastTimestamp, long?toNewTimestamp, Guid?excludingScopeId,
bool supportsMultiActiveResultSets, string batchRootDirectory, string batchDirectoryName,
DbConnection connection, DbTransaction transaction,
CancellationToken cancellationToken, IProgress <ProgressArgs> progress)
{
// batch info containing changes
BatchInfo batchInfo;
// Statistics about changes that are selected
DatabaseChangesSelected changesSelected;
context.SyncStage = SyncStage.ChangesSelecting;
if (context.SyncWay == SyncWay.Upload && context.SyncType == SyncType.Reinitialize)
{
(batchInfo, changesSelected) = await this.InternalGetEmptyChangesAsync(scopeInfo, batchRootDirectory).ConfigureAwait(false);
return(context, batchInfo, changesSelected);
}
// create local directory
if (!string.IsNullOrEmpty(batchRootDirectory) && !Directory.Exists(batchRootDirectory))
{
Directory.CreateDirectory(batchRootDirectory);
}
changesSelected = new DatabaseChangesSelected();
// Create a batch
// batchinfo generate a schema clone with scope columns if needed
batchInfo = new BatchInfo(scopeInfo.Schema, batchRootDirectory, batchDirectoryName);
batchInfo.TryRemoveDirectory();
batchInfo.CreateDirectory();
// Call interceptor
var databaseChangesSelectingArgs = new DatabaseChangesSelectingArgs(context, batchInfo.GetDirectoryFullPath(), this.Options.BatchSize, isNew,
fromLastTimestamp, toNewTimestamp, connection, transaction);
await this.InterceptAsync(databaseChangesSelectingArgs, progress, cancellationToken).ConfigureAwait(false);
var cptSyncTable = 0;
var currentProgress = context.ProgressPercentage;
var schemaTables = scopeInfo.Schema.Tables.SortByDependencies(tab => tab.GetRelations().Select(r => r.GetParentTable()));
var lstAllBatchPartInfos = new ConcurrentBag <BatchPartInfo>();
var lstTableChangesSelected = new ConcurrentBag <TableChangesSelected>();
var threadNumberLimits = supportsMultiActiveResultSets ? 16 : 1;
if (supportsMultiActiveResultSets)
{
await schemaTables.ForEachAsync(async syncTable =>
{
if (cancellationToken.IsCancellationRequested)
{
return;
}
// tmp count of table for report progress pct
cptSyncTable++;
List <BatchPartInfo> syncTableBatchPartInfos;
TableChangesSelected tableChangesSelected;
(context, syncTableBatchPartInfos, tableChangesSelected) = await InternalReadSyncTableChangesAsync(
scopeInfo, context, excludingScopeId, syncTable, batchInfo, isNew, fromLastTimestamp, connection, transaction, cancellationToken, progress).ConfigureAwait(false);
if (syncTableBatchPartInfos == null)
{
return;
}
// We don't report progress if no table changes is empty, to limit verbosity
if (tableChangesSelected != null && (tableChangesSelected.Deletes > 0 || tableChangesSelected.Upserts > 0))
{
lstTableChangesSelected.Add(tableChangesSelected);
}
// Add sync table bpi to all bpi
syncTableBatchPartInfos.ForEach(bpi => lstAllBatchPartInfos.Add(bpi));
context.ProgressPercentage = currentProgress + (cptSyncTable * 0.2d / scopeInfo.Schema.Tables.Count);
}, threadNumberLimits);
}
else
{
foreach (var syncTable in schemaTables)
{
if (cancellationToken.IsCancellationRequested)
{
continue;
}
// tmp count of table for report progress pct
cptSyncTable++;
List <BatchPartInfo> syncTableBatchPartInfos;
TableChangesSelected tableChangesSelected;
(context, syncTableBatchPartInfos, tableChangesSelected) = await InternalReadSyncTableChangesAsync(
scopeInfo, context, excludingScopeId, syncTable, batchInfo, isNew, fromLastTimestamp, connection, transaction, cancellationToken, progress).ConfigureAwait(false);
if (syncTableBatchPartInfos == null)
{
continue;
}
// We don't report progress if no table changes is empty, to limit verbosity
if (tableChangesSelected != null && (tableChangesSelected.Deletes > 0 || tableChangesSelected.Upserts > 0))
{
lstTableChangesSelected.Add(tableChangesSelected);
}
// Add sync table bpi to all bpi
syncTableBatchPartInfos.ForEach(bpi => lstAllBatchPartInfos.Add(bpi));
context.ProgressPercentage = currentProgress + (cptSyncTable * 0.2d / scopeInfo.Schema.Tables.Count);
}
}
while (!lstTableChangesSelected.IsEmpty)
{
if (lstTableChangesSelected.TryTake(out var tableChangesSelected))
{
changesSelected.TableChangesSelected.Add(tableChangesSelected);
}
}
// delete all empty batchparts (empty tables)
foreach (var bpi in lstAllBatchPartInfos.Where(bpi => bpi.RowsCount <= 0))
{
File.Delete(Path.Combine(batchInfo.GetDirectoryFullPath(), bpi.FileName));
}
// Generate a good index order to be compliant with previous versions
var tmpLstBatchPartInfos = new List <BatchPartInfo>();
foreach (var table in schemaTables)
{
// get all bpi where count > 0 and ordered by index
foreach (var bpi in lstAllBatchPartInfos.Where(bpi => bpi.RowsCount > 0 && bpi.Tables[0].EqualsByName(new BatchPartTableInfo(table.TableName, table.SchemaName))).OrderBy(bpi => bpi.Index).ToArray())
{
batchInfo.BatchPartsInfo.Add(bpi);
batchInfo.RowsCount += bpi.RowsCount;
tmpLstBatchPartInfos.Add(bpi);
}
}
var newBatchIndex = 0;
foreach (var bpi in tmpLstBatchPartInfos)
{
bpi.Index = newBatchIndex;
newBatchIndex++;
bpi.IsLastBatch = newBatchIndex == tmpLstBatchPartInfos.Count;
}
//Set the total rows count contained in the batch info
batchInfo.EnsureLastBatch();
if (batchInfo.RowsCount <= 0)
{
var cleanFolder = await this.InternalCanCleanFolderAsync(scopeInfo.Name, context.Parameters, batchInfo, cancellationToken).ConfigureAwait(false);
batchInfo.Clear(cleanFolder);
}
var databaseChangesSelectedArgs = new DatabaseChangesSelectedArgs(context, fromLastTimestamp, toNewTimestamp, batchInfo, changesSelected, connection);
await this.InterceptAsync(databaseChangesSelectedArgs, progress, cancellationToken).ConfigureAwait(false);
return(context, batchInfo, changesSelected);
}
