/// <summary>
/// Generate an empty BatchInfo
/// </summary>
internal async Task <(BatchInfo, DatabaseChangesSelected)> GetEmptyChangesAsync(MessageGetChangesBatch message)
{
// Get config
var isBatched = message.BatchSize > 0;
// create the in memory changes set
var changesSet = new SyncSet(message.Schema.ScopeName);
// Create a Schema set without readonly tables, attached to memory changes
foreach (var table in message.Schema.Tables)
{
DbSyncAdapter.CreateChangesTable(message.Schema.Tables[table.TableName, table.SchemaName], changesSet);
}
// Create the batch info, in memory
var batchInfo = new BatchInfo(!isBatched, changesSet, message.BatchDirectory);;
// add changes to batchInfo
await batchInfo.AddChangesAsync(new SyncSet()).ConfigureAwait(false);
// Create a new empty in-memory batch info
return(batchInfo, new DatabaseChangesSelected());
}
/// <summary>
/// Gets a batch of changes to synchronize when given batch size,
/// destination knowledge, and change data retriever parameters.
/// </summary>
/// <returns>A DbSyncContext object that will be used to retrieve the modified data.</returns>
internal virtual async Task <(SyncContext, BatchInfo, DatabaseChangesSelected)> InternalGetChangesAsync(
SyncContext context, MessageGetChangesBatch message,
DbConnection connection, DbTransaction transaction,
CancellationToken cancellationToken, IProgress <ProgressArgs> progress)
{
// batch info containing changes
BatchInfo batchInfo;
// Statistics about changes that are selected
DatabaseChangesSelected changesSelected;
if (context.SyncWay == SyncWay.Upload && context.SyncType == SyncType.Reinitialize)
{
(batchInfo, changesSelected) = await this.InternalGetEmptyChangesAsync(message).ConfigureAwait(false);
return(context, batchInfo, changesSelected);
}
// Call interceptor
await this.InterceptAsync(new DatabaseChangesSelectingArgs(context, message, connection, transaction), cancellationToken).ConfigureAwait(false);
// create local directory
if (message.BatchSize > 0 && !string.IsNullOrEmpty(message.BatchDirectory) && !Directory.Exists(message.BatchDirectory))
{
Directory.CreateDirectory(message.BatchDirectory);
}
changesSelected = new DatabaseChangesSelected();
// numbers of batch files generated
var batchIndex = 0;
// Check if we are in batch mode
var isBatch = message.BatchSize > 0;
// Create a batch info in memory (if !isBatch) or serialized on disk (if isBatch)
// batchinfo generate a schema clone with scope columns if needed
batchInfo = new BatchInfo(!isBatch, message.Schema, message.BatchDirectory);
// Clean SyncSet, we will add only tables we need in the batch info
var changesSet = new SyncSet();
var cptSyncTable = 0;
var currentProgress = context.ProgressPercentage;
foreach (var syncTable in message.Schema.Tables)
{
// tmp count of table for report progress pct
cptSyncTable++;
// Only table schema is replicated, no datas are applied
if (syncTable.SyncDirection == SyncDirection.None)
{
continue;
}
// if we are in upload stage, so check if table is not download only
if (context.SyncWay == SyncWay.Upload && syncTable.SyncDirection == SyncDirection.DownloadOnly)
{
continue;
}
// if we are in download stage, so check if table is not download only
if (context.SyncWay == SyncWay.Download && syncTable.SyncDirection == SyncDirection.UploadOnly)
{
continue;
}
// Get Command
var selectIncrementalChangesCommand = await this.GetSelectChangesCommandAsync(context, syncTable, message.Setup, message.IsNew, connection, transaction);
// Set parameters
this.SetSelectChangesCommonParameters(context, syncTable, message.ExcludingScopeId, message.IsNew, message.LastTimestamp, selectIncrementalChangesCommand);
// launch interceptor if any
var args = new TableChangesSelectingArgs(context, syncTable, selectIncrementalChangesCommand, connection, transaction);
await this.InterceptAsync(args, cancellationToken).ConfigureAwait(false);
if (!args.Cancel && args.Command != null)
{
// Statistics
var tableChangesSelected = new TableChangesSelected(syncTable.TableName, syncTable.SchemaName);
// Create a chnages table with scope columns
var changesSetTable = DbSyncAdapter.CreateChangesTable(message.Schema.Tables[syncTable.TableName, syncTable.SchemaName], changesSet);
// Get the reader
using var dataReader = await args.Command.ExecuteReaderAsync().ConfigureAwait(false);
// memory size total
double rowsMemorySize = 0L;
while (dataReader.Read())
{
// Create a row from dataReader
var row = CreateSyncRowFromReader(dataReader, changesSetTable);
// Add the row to the changes set
changesSetTable.Rows.Add(row);
// Set the correct state to be applied
if (row.RowState == DataRowState.Deleted)
{
tableChangesSelected.Deletes++;
}
else if (row.RowState == DataRowState.Modified)
{
tableChangesSelected.Upserts++;
}
// calculate row size if in batch mode
if (isBatch)
{
var fieldsSize = ContainerTable.GetRowSizeFromDataRow(row.ToArray());
var finalFieldSize = fieldsSize / 1024d;
if (finalFieldSize > message.BatchSize)
{
throw new RowOverSizedException(finalFieldSize.ToString());
}
// Calculate the new memory size
rowsMemorySize += finalFieldSize;
// Next line if we don't reach the batch size yet.
if (rowsMemorySize <= message.BatchSize)
{
continue;
}
// Check interceptor
var batchTableChangesSelectedArgs = new TableChangesSelectedArgs(context, changesSetTable, tableChangesSelected, connection, transaction);
await this.InterceptAsync(batchTableChangesSelectedArgs, cancellationToken).ConfigureAwait(false);
// add changes to batchinfo
await batchInfo.AddChangesAsync(changesSet, batchIndex, false, message.SerializerFactory, this).ConfigureAwait(false);
// increment batch index
batchIndex++;
// we know the datas are serialized here, so we can flush the set
changesSet.Clear();
// Recreate an empty ContainerSet and a ContainerTable
changesSet = new SyncSet();
changesSetTable = DbSyncAdapter.CreateChangesTable(message.Schema.Tables[syncTable.TableName, syncTable.SchemaName], changesSet);
// Init the row memory size
rowsMemorySize = 0L;
}
}
dataReader.Close();
// We don't report progress if no table changes is empty, to limit verbosity
if (tableChangesSelected.Deletes > 0 || tableChangesSelected.Upserts > 0)
{
changesSelected.TableChangesSelected.Add(tableChangesSelected);
}
// even if no rows raise the interceptor
var tableChangesSelectedArgs = new TableChangesSelectedArgs(context, changesSetTable, tableChangesSelected, connection, transaction);
await this.InterceptAsync(tableChangesSelectedArgs, cancellationToken).ConfigureAwait(false);
context.ProgressPercentage = currentProgress + (cptSyncTable * 0.2d / message.Schema.Tables.Count);
// only raise report progress if we have something
if (tableChangesSelectedArgs.TableChangesSelected.TotalChanges > 0)
{
this.ReportProgress(context, progress, tableChangesSelectedArgs);
}
}
}
// We are in batch mode, and we are at the last batchpart info
// Even if we don't have rows inside, we return the changesSet, since it contains at least schema
if (changesSet != null && changesSet.HasTables && changesSet.HasRows)
{
await batchInfo.AddChangesAsync(changesSet, batchIndex, true, message.SerializerFactory, this).ConfigureAwait(false);
}
//Set the total rows count contained in the batch info
batchInfo.RowsCount = changesSelected.TotalChangesSelected;
// Check the last index as the last batch
batchInfo.EnsureLastBatch();
// Raise database changes selected
if (changesSelected.TotalChangesSelected > 0 || changesSelected.TotalChangesSelectedDeletes > 0 || changesSelected.TotalChangesSelectedUpdates > 0)
{
var databaseChangesSelectedArgs = new DatabaseChangesSelectedArgs(context, message.LastTimestamp, batchInfo, changesSelected, connection);
this.ReportProgress(context, progress, databaseChangesSelectedArgs);
await this.InterceptAsync(databaseChangesSelectedArgs, cancellationToken).ConfigureAwait(false);
}
return(context, batchInfo, changesSelected);
}
/// <summary>
/// update configuration object with tables desc from server database
/// </summary>
public virtual async Task <(SyncContext, BatchInfo)> CreateSnapshotAsync(SyncContext context, SyncSet schema, SyncSetup setup,
DbConnection connection, DbTransaction transaction, string snapshotDirectory, int batchSize, long remoteClientTimestamp,
CancellationToken cancellationToken, IProgress <ProgressArgs> progress = null)
{
// create local directory
if (!Directory.Exists(snapshotDirectory))
{
this.Orchestrator.logger.LogDebug(SyncEventsId.CreateDirectory, new { SnapshotDirectory = snapshotDirectory });
Directory.CreateDirectory(snapshotDirectory);
}
// cleansing scope name
var directoryScopeName = new string(context.ScopeName.Where(char.IsLetterOrDigit).ToArray());
var directoryFullPath = Path.Combine(snapshotDirectory, directoryScopeName);
// create local directory with scope inside
if (!Directory.Exists(directoryFullPath))
{
this.Orchestrator.logger.LogDebug(SyncEventsId.CreateDirectory, new { DirectoryFullPath = directoryFullPath });
Directory.CreateDirectory(directoryFullPath);
}
// numbers of batch files generated
var batchIndex = 0;
// create the in memory changes set
var changesSet = new SyncSet();
var sb = new StringBuilder();
var underscore = "";
if (context.Parameters != null)
{
foreach (var p in context.Parameters.OrderBy(p => p.Name))
{
var cleanValue = new string(p.Value.ToString().Where(char.IsLetterOrDigit).ToArray());
var cleanName = new string(p.Name.Where(char.IsLetterOrDigit).ToArray());
sb.Append($"{underscore}{cleanName}_{cleanValue}");
underscore = "_";
}
}
var directoryName = sb.ToString();
directoryName = string.IsNullOrEmpty(directoryName) ? "ALL" : directoryName;
// batchinfo generate a schema clone with scope columns if needed
var batchInfo = new BatchInfo(false, schema, directoryFullPath, directoryName);
// Delete directory if already exists
directoryFullPath = Path.Combine(directoryFullPath, directoryName);
if (Directory.Exists(directoryFullPath))
{
this.Orchestrator.logger.LogDebug(SyncEventsId.DropDirectory, new { DirectoryFullPath = directoryFullPath });
Directory.Delete(directoryFullPath, true);
}
foreach (var syncTable in schema.Tables)
{
var tableBuilder = this.GetTableBuilder(syncTable, setup);
var syncAdapter = tableBuilder.CreateSyncAdapter();
// launch interceptor if any
await this.Orchestrator.InterceptAsync(new TableChangesSelectingArgs(context, syncTable, connection, transaction), cancellationToken).ConfigureAwait(false);
// Get Select initialize changes command
var selectIncrementalChangesCommand = await this.GetSelectChangesCommandAsync(context, syncAdapter, syncTable, true, connection, transaction);
// Set parameters
this.SetSelectChangesCommonParameters(context, syncTable, null, true, 0, selectIncrementalChangesCommand);
// log
this.Orchestrator.logger.LogDebug(SyncEventsId.CreateSnapshot, new
{
SelectChangesCommandText = selectIncrementalChangesCommand.CommandText,
ExcludingScopeId = Guid.Empty,
IsNew = true,
LastTimestamp = 0
});
// Get the reader
using (var dataReader = await selectIncrementalChangesCommand.ExecuteReaderAsync().ConfigureAwait(false))
{
// memory size total
double rowsMemorySize = 0L;
// Create a chnages table with scope columns
var changesSetTable = SyncAdapter.CreateChangesTable(schema.Tables[syncTable.TableName, syncTable.SchemaName], changesSet);
while (dataReader.Read())
{
// Create a row from dataReader
var row = CreateSyncRowFromReader(dataReader, changesSetTable);
// Add the row to the changes set
changesSetTable.Rows.Add(row);
// Log trace row
this.Orchestrator.logger.LogTrace(SyncEventsId.CreateSnapshot, row);
var fieldsSize = ContainerTable.GetRowSizeFromDataRow(row.ToArray());
var finalFieldSize = fieldsSize / 1024d;
if (finalFieldSize > batchSize)
{
throw new RowOverSizedException(finalFieldSize.ToString());
}
// Calculate the new memory size
rowsMemorySize += finalFieldSize;
// Next line if we don't reach the batch size yet.
if (rowsMemorySize <= batchSize)
{
continue;
}
// add changes to batchinfo
await batchInfo.AddChangesAsync(changesSet, batchIndex, false, this.Orchestrator).ConfigureAwait(false);
this.Orchestrator.logger.LogDebug(SyncEventsId.CreateBatch, changesSet);
// increment batch index
batchIndex++;
// we know the datas are serialized here, so we can flush the set
changesSet.Clear();
// Recreate an empty ContainerSet and a ContainerTable
changesSet = new SyncSet();
changesSetTable = SyncAdapter.CreateChangesTable(schema.Tables[syncTable.TableName, syncTable.SchemaName], changesSet);
// Init the row memory size
rowsMemorySize = 0L;
}
}
//selectIncrementalChangesCommand.Dispose();
}
if (changesSet != null && changesSet.HasTables)
{
await batchInfo.AddChangesAsync(changesSet, batchIndex, true, this.Orchestrator).ConfigureAwait(false);
this.Orchestrator.logger.LogDebug(SyncEventsId.CreateBatch, changesSet);
}
// Check the last index as the last batch
batchInfo.EnsureLastBatch();
batchInfo.Timestamp = remoteClientTimestamp;
// Serialize on disk.
var jsonConverter = new JsonConverter <BatchInfo>();
var summaryFileName = Path.Combine(directoryFullPath, "summary.json");
using (var f = new FileStream(summaryFileName, FileMode.CreateNew, FileAccess.ReadWrite))
{
this.Orchestrator.logger.LogDebug(SyncEventsId.CreateSnapshotSummary, batchInfo);
var bytes = await jsonConverter.SerializeAsync(batchInfo).ConfigureAwait(false);
f.Write(bytes, 0, bytes.Length);
}
return(context, batchInfo);
}
GetChangesAsync(ScopeInfo clientScope, CancellationToken cancellationToken = default, IProgress <ProgressArgs> progress = null)
{
SyncSet schema;
// Get context or create a new one
var ctx = this.GetContext();
if (!this.StartTime.HasValue)
{
this.StartTime = DateTime.UtcNow;
}
ServerScopeInfo serverScopeInfo;
// Need the server scope
serverScopeInfo = await this.EnsureSchemaAsync(cancellationToken, progress).ConfigureAwait(false);
schema = serverScopeInfo.Schema;
schema.EnsureSchema();
clientScope.Schema = schema;
clientScope.Setup = serverScopeInfo.Setup;
clientScope.Version = serverScopeInfo.Version;
// generate a message to send
var changesToSend = new HttpMessageSendChangesRequest(ctx, clientScope)
{
Changes = null,
IsLastBatch = true,
BatchIndex = 0
};
var serializer = this.SerializerFactory.GetSerializer <HttpMessageSendChangesRequest>();
var binaryData = await serializer.SerializeAsync(changesToSend);
await this.InterceptAsync(new HttpMessageSendChangesRequestArgs(binaryData), cancellationToken).ConfigureAwait(false);
// response
var httpMessageContent = await this.httpRequestHandler.ProcessRequestAsync <HttpMessageSendChangesResponse>
(this.HttpClient, this.ServiceUri, binaryData, HttpStep.GetChanges, ctx.SessionId, clientScope.Name,
this.SerializerFactory, this.Converter, this.Options.BatchSize, cancellationToken).ConfigureAwait(false);
// if nothing available, return empty response
if (httpMessageContent.Changes == null)
{
return(httpMessageContent.RemoteClientTimestamp, null, new DatabaseChangesSelected());
}
// Get if we need to work in memory or serialize things
var workInMemoryLocally = this.Options.BatchSize == 0;
bool isLastBatch;
//timestamp generated by the server, hold in the client db
long remoteClientTimestamp;
// Create the BatchInfo and SyncContext to return at the end
var serverBatchInfo = new BatchInfo(workInMemoryLocally, schema, this.Options.BatchDirectory);
// stats
DatabaseChangesSelected serverChangesSelected;
// While we are not reaching the last batch from server
do
{
// Check if we are at the last batch.
// If so, we won't make another loop
isLastBatch = httpMessageContent.IsLastBatch;
ctx = httpMessageContent.SyncContext;
remoteClientTimestamp = httpMessageContent.RemoteClientTimestamp;
serverChangesSelected = httpMessageContent.ServerChangesSelected;
var changesSet = serverBatchInfo.SanitizedSchema.Clone();
changesSet.ImportContainerSet(httpMessageContent.Changes, false);
if (this.Converter != null && changesSet.HasRows)
{
AfterDeserializedRows(changesSet);
}
// Create a BatchPartInfo instance
await serverBatchInfo.AddChangesAsync(changesSet, httpMessageContent.BatchIndex, isLastBatch, this);
// free some memory
if (httpMessageContent.Changes != null)
{
httpMessageContent.Changes.Clear();
}
if (!isLastBatch)
{
// Ask for the next batch index
var requestBatchIndex = httpMessageContent.BatchIndex + 1;
// Create the message enveloppe
var httpMessage = new HttpMessageGetMoreChangesRequest(ctx, requestBatchIndex);
// serialize message
var serializer2 = this.SerializerFactory.GetSerializer <HttpMessageGetMoreChangesRequest>();
var binaryData2 = await serializer2.SerializeAsync(httpMessage);
await this.InterceptAsync(new HttpMessageGetMoreChangesRequestArgs(binaryData), cancellationToken).ConfigureAwait(false);
httpMessageContent = await this.httpRequestHandler.ProcessRequestAsync <HttpMessageSendChangesResponse>(
this.HttpClient, this.ServiceUri, binaryData2, HttpStep.GetMoreChanges, ctx.SessionId, this.ScopeName,
this.SerializerFactory, this.Converter, 0, cancellationToken).ConfigureAwait(false);
}
} while (!isLastBatch);
// generate the new scope item
this.CompleteTime = DateTime.UtcNow;
// Reaffect context
this.SetContext(httpMessageContent.SyncContext);
return(remoteClientTimestamp, serverBatchInfo, serverChangesSelected);
}
GetSnapshotAsync(SyncSet schema = null, CancellationToken cancellationToken = default, IProgress <ProgressArgs> progress = null)
{
// Get context or create a new one
var ctx = this.GetContext();
if (!this.StartTime.HasValue)
{
this.StartTime = DateTime.UtcNow;
}
// Make a remote call to get Schema from remote provider
if (schema == null)
{
var serverScopeInfo = await this.EnsureSchemaAsync(cancellationToken, progress).ConfigureAwait(false);
schema = serverScopeInfo.Schema;
schema.EnsureSchema();
}
// Create the BatchInfo and SyncContext to return at the end
// Set InMemory by default to "true", but the real value is coming from server side
var serverBatchInfo = new BatchInfo(false, schema, this.Options.BatchDirectory);
bool isLastBatch;
//timestamp generated by the server, hold in the client db
long remoteClientTimestamp;
// generate a message to send
var changesToSend = new HttpMessageSendChangesRequest(ctx, null)
{
Changes = null,
IsLastBatch = true,
BatchIndex = 0
};
var serializer = this.SerializerFactory.GetSerializer <HttpMessageSendChangesRequest>();
var binaryData = await serializer.SerializeAsync(changesToSend);
var httpMessageContent = await this.httpRequestHandler.ProcessRequestAsync <HttpMessageSendChangesResponse>(
this.HttpClient, this.ServiceUri, binaryData, HttpStep.GetSnapshot, ctx.SessionId, this.ScopeName,
this.SerializerFactory, this.Converter, 0, cancellationToken).ConfigureAwait(false);
// if no snapshot available, return empty response
if (httpMessageContent.Changes == null)
{
return(httpMessageContent.RemoteClientTimestamp, null);
}
// While we are not reaching the last batch from server
do
{
// Check if we are at the last batch.
// If so, we won't make another loop
isLastBatch = httpMessageContent.IsLastBatch;
ctx = httpMessageContent.SyncContext;
remoteClientTimestamp = httpMessageContent.RemoteClientTimestamp;
var changesSet = serverBatchInfo.SanitizedSchema.Clone();
changesSet.ImportContainerSet(httpMessageContent.Changes, false);
if (this.Converter != null && changesSet.HasRows)
{
AfterDeserializedRows(changesSet);
}
// Create a BatchPartInfo instance
await serverBatchInfo.AddChangesAsync(changesSet, httpMessageContent.BatchIndex, isLastBatch, this);
// free some memory
if (httpMessageContent.Changes != null)
{
httpMessageContent.Changes.Clear();
}
if (!isLastBatch)
{
// Ask for the next batch index
var requestBatchIndex = httpMessageContent.BatchIndex + 1;
// Create the message enveloppe
var httpMessage = new HttpMessageGetMoreChangesRequest(ctx, requestBatchIndex);
// serialize message
var serializer2 = this.SerializerFactory.GetSerializer <HttpMessageGetMoreChangesRequest>();
var binaryData2 = await serializer2.SerializeAsync(httpMessage);
await this.InterceptAsync(new HttpMessageGetMoreChangesRequestArgs(binaryData), cancellationToken).ConfigureAwait(false);
httpMessageContent = await this.httpRequestHandler.ProcessRequestAsync <HttpMessageSendChangesResponse>(
this.HttpClient, this.ServiceUri, binaryData2, HttpStep.GetMoreChanges, ctx.SessionId, this.ScopeName,
this.SerializerFactory, this.Converter, 0, cancellationToken).ConfigureAwait(false);
}
} while (!isLastBatch);
// Reaffect context
this.SetContext(httpMessageContent.SyncContext);
return(remoteClientTimestamp, serverBatchInfo);
}
ApplyThenGetChangesAsync(ScopeInfo scope, BatchInfo clientBatchInfo, CancellationToken cancellationToken = default, IProgress <ProgressArgs> progress = null)
{
SyncSet schema;
// Get context or create a new one
var ctx = this.GetContext();
if (!this.StartTime.HasValue)
{
this.StartTime = DateTime.UtcNow;
}
//// create the in memory changes set
//var changesSet = new SyncSet();
// is it something that could happens ?
if (scope.Schema == null)
{
// Make a remote call to get Schema from remote provider
var serverScopeInfo = await this.EnsureSchemaAsync(cancellationToken, progress).ConfigureAwait(false);
schema = serverScopeInfo.Schema;
}
else
{
schema = scope.Schema;
}
schema.EnsureSchema();
// if we don't have any BatchPartsInfo, just generate a new one to get, at least, something to send to the server
// and get a response with new data from server
if (clientBatchInfo == null)
{
clientBatchInfo = new BatchInfo(true, schema);
}
// Get sanitized schema, without readonly columns
var sanitizedSchema = clientBatchInfo.SanitizedSchema;
// --------------------------------------------------------------
// STEP 1 : Send everything to the server side
// --------------------------------------------------------------
// response
HttpMessageSendChangesResponse httpMessageContent = null;
// If not in memory and BatchPartsInfo.Count == 0, nothing to send.
// But we need to send something, so generate a little batch part
if (clientBatchInfo.InMemory || (!clientBatchInfo.InMemory && clientBatchInfo.BatchPartsInfo.Count == 0))
{
var changesToSend = new HttpMessageSendChangesRequest(ctx, scope);
if (this.Converter != null && clientBatchInfo.InMemoryData != null && clientBatchInfo.InMemoryData.HasRows)
{
this.BeforeSerializeRows(clientBatchInfo.InMemoryData);
}
var containerSet = clientBatchInfo.InMemoryData == null ? new ContainerSet() : clientBatchInfo.InMemoryData.GetContainerSet();
changesToSend.Changes = containerSet;
changesToSend.IsLastBatch = true;
changesToSend.BatchIndex = 0;
// serialize message
var serializer = this.SerializerFactory.GetSerializer <HttpMessageSendChangesRequest>();
var binaryData = await serializer.SerializeAsync(changesToSend);
await this.InterceptAsync(new HttpMessageSendChangesRequestArgs(binaryData), cancellationToken).ConfigureAwait(false);
httpMessageContent = await this.httpRequestHandler.ProcessRequestAsync <HttpMessageSendChangesResponse>
(this.HttpClient, this.ServiceUri, binaryData, HttpStep.SendChanges, ctx.SessionId, scope.Name,
this.SerializerFactory, this.Converter, this.Options.BatchSize, cancellationToken).ConfigureAwait(false);
}
else
{
// Foreach part, will have to send them to the remote
// once finished, return context
foreach (var bpi in clientBatchInfo.BatchPartsInfo.OrderBy(bpi => bpi.Index))
{
// If BPI is InMempory, no need to deserialize from disk
// othewise load it
await bpi.LoadBatchAsync(sanitizedSchema, clientBatchInfo.GetDirectoryFullPath(), this);
var changesToSend = new HttpMessageSendChangesRequest(ctx, scope);
if (this.Converter != null && bpi.Data.HasRows)
{
BeforeSerializeRows(bpi.Data);
}
// Set the change request properties
changesToSend.Changes = bpi.Data.GetContainerSet();
changesToSend.IsLastBatch = bpi.IsLastBatch;
changesToSend.BatchIndex = bpi.Index;
// serialize message
var serializer = this.SerializerFactory.GetSerializer <HttpMessageSendChangesRequest>();
var binaryData = await serializer.SerializeAsync(changesToSend);
await this.InterceptAsync(new HttpMessageSendChangesRequestArgs(binaryData), cancellationToken).ConfigureAwait(false);
httpMessageContent = await this.httpRequestHandler.ProcessRequestAsync <HttpMessageSendChangesResponse>
(this.HttpClient, this.ServiceUri, binaryData, HttpStep.SendChanges, ctx.SessionId, scope.Name,
this.SerializerFactory, this.Converter, this.Options.BatchSize, cancellationToken).ConfigureAwait(false);
// for some reasons, if server don't want to wait for more, just break
// That should never happened, actually
if (httpMessageContent.ServerStep != HttpStep.SendChangesInProgress)
{
break;
}
}
}
// --------------------------------------------------------------
// STEP 2 : Receive everything from the server side
// --------------------------------------------------------------
// Now we have sent all the datas to the server and now :
// We have a FIRST response from the server with new datas
// 1) Could be the only one response (enough or InMemory is set on the server side)
// 2) Could bt the first response and we need to download all batchs
// While we have an other batch to process
var isLastBatch = false;
// Get if we need to work in memory or serialize things
var workInMemoryLocally = this.Options.BatchSize == 0;
// Create the BatchInfo and SyncContext to return at the end
// Set InMemory by default to "true", but the real value is coming from server side
var serverBatchInfo = new BatchInfo(workInMemoryLocally, schema, this.Options.BatchDirectory);
// stats
DatabaseChangesSelected serverChangesSelected = null;
DatabaseChangesApplied clientChangesApplied = null;
//timestamp generated by the server, hold in the client db
long remoteClientTimestamp = 0;
// While we are not reaching the last batch from server
do
{
// Check if we are at the last batch.
// If so, we won't make another loop
isLastBatch = httpMessageContent.IsLastBatch;
serverChangesSelected = httpMessageContent.ServerChangesSelected;
clientChangesApplied = httpMessageContent.ClientChangesApplied;
ctx = httpMessageContent.SyncContext;
remoteClientTimestamp = httpMessageContent.RemoteClientTimestamp;
var changesSet = sanitizedSchema.Clone();
changesSet.ImportContainerSet(httpMessageContent.Changes, false);
if (this.Converter != null && changesSet.HasRows)
{
AfterDeserializedRows(changesSet);
}
// Create a BatchPartInfo instance
await serverBatchInfo.AddChangesAsync(changesSet, httpMessageContent.BatchIndex, isLastBatch, this);
// free some memory
if (!workInMemoryLocally && httpMessageContent.Changes != null)
{
httpMessageContent.Changes.Clear();
}
if (!isLastBatch)
{
// Ask for the next batch index
var requestBatchIndex = httpMessageContent.BatchIndex + 1;
// Create the message enveloppe
var httpMessage = new HttpMessageGetMoreChangesRequest(ctx, requestBatchIndex);
// serialize message
var serializer = this.SerializerFactory.GetSerializer <HttpMessageGetMoreChangesRequest>();
var binaryData = await serializer.SerializeAsync(httpMessage);
await this.InterceptAsync(new HttpMessageGetMoreChangesRequestArgs(binaryData), cancellationToken).ConfigureAwait(false);
httpMessageContent = await this.httpRequestHandler.ProcessRequestAsync <HttpMessageSendChangesResponse>(
this.HttpClient, this.ServiceUri, binaryData, HttpStep.GetMoreChanges, ctx.SessionId, scope.Name,
this.SerializerFactory, this.Converter, this.Options.BatchSize, cancellationToken).ConfigureAwait(false);
}
} while (!isLastBatch);
// generate the new scope item
this.CompleteTime = DateTime.UtcNow;
// Reaffect context
this.SetContext(httpMessageContent.SyncContext);
return(remoteClientTimestamp, serverBatchInfo,
httpMessageContent.ConflictResolutionPolicy, clientChangesApplied, serverChangesSelected);
}
/// <summary>
/// Gets a batch of changes to synchronize when given batch size,
/// destination knowledge, and change data retriever parameters.
/// </summary>
/// <returns>A DbSyncContext object that will be used to retrieve the modified data.</returns>
public virtual async Task <(SyncContext, BatchInfo, DatabaseChangesSelected)> GetChangeBatchAsync(
SyncContext context, MessageGetChangesBatch message,
DbConnection connection, DbTransaction transaction,
CancellationToken cancellationToken, IProgress <ProgressArgs> progress = null)
{
// batch info containing changes
BatchInfo batchInfo;
// Statistics about changes that are selected
DatabaseChangesSelected changesSelected;
if (context.SyncWay == SyncWay.Upload && context.SyncType == SyncType.Reinitialize)
{
(batchInfo, changesSelected) = await this.GetEmptyChangesAsync(message);
return(context, batchInfo, changesSelected);
}
// Check if the provider is not outdated
var isOutdated = this.IsRemoteOutdated();
// Get a chance to make the sync even if it's outdated
if (isOutdated)
{
var outdatedArgs = new OutdatedArgs(context, null, null);
// Interceptor
await this.InterceptAsync(outdatedArgs).ConfigureAwait(false);
if (outdatedArgs.Action != OutdatedAction.Rollback)
{
context.SyncType = outdatedArgs.Action == OutdatedAction.Reinitialize ? SyncType.Reinitialize : SyncType.ReinitializeWithUpload;
}
if (outdatedArgs.Action == OutdatedAction.Rollback)
{
throw new OutOfDateException();
}
}
// create local directory
if (message.BatchSize > 0 && !string.IsNullOrEmpty(message.BatchDirectory) && !Directory.Exists(message.BatchDirectory))
{
Directory.CreateDirectory(message.BatchDirectory);
}
// numbers of batch files generated
var batchIndex = 0;
// Check if we are in batch mode
var isBatch = message.BatchSize > 0;
// Create stats object to store changes count
var changes = new DatabaseChangesSelected();
// create the in memory changes set
var changesSet = new SyncSet(message.Schema.ScopeName);
// Create a Schema set without readonly columns, attached to memory changes
foreach (var table in message.Schema.Tables)
{
DbSyncAdapter.CreateChangesTable(message.Schema.Tables[table.TableName, table.SchemaName], changesSet);
}
// Create a batch info in memory (if !isBatch) or serialized on disk (if isBatch)
// batchinfo generate a schema clone with scope columns if needed
batchInfo = new BatchInfo(!isBatch, changesSet, message.BatchDirectory);
// Clear tables, we will add only the ones we need in the batch info
changesSet.Clear();
foreach (var syncTable in message.Schema.Tables)
{
// if we are in upload stage, so check if table is not download only
if (context.SyncWay == SyncWay.Upload && syncTable.SyncDirection == SyncDirection.DownloadOnly)
{
continue;
}
// if we are in download stage, so check if table is not download only
if (context.SyncWay == SyncWay.Download && syncTable.SyncDirection == SyncDirection.UploadOnly)
{
continue;
}
var tableBuilder = this.GetTableBuilder(syncTable);
var syncAdapter = tableBuilder.CreateSyncAdapter(connection, transaction);
// raise before event
context.SyncStage = SyncStage.TableChangesSelecting;
var tableChangesSelectingArgs = new TableChangesSelectingArgs(context, syncTable.TableName, connection, transaction);
// launch interceptor if any
await this.InterceptAsync(tableChangesSelectingArgs).ConfigureAwait(false);
// Get Command
var selectIncrementalChangesCommand = this.GetSelectChangesCommand(context, syncAdapter, syncTable, message.IsNew);
// Set parameters
this.SetSelectChangesCommonParameters(context, syncTable, message.ExcludingScopeId, message.IsNew, message.LastTimestamp, selectIncrementalChangesCommand);
// Statistics
var tableChangesSelected = new TableChangesSelected(syncTable.TableName);
// Get the reader
using (var dataReader = selectIncrementalChangesCommand.ExecuteReader())
{
// memory size total
double rowsMemorySize = 0L;
// Create a chnages table with scope columns
var changesSetTable = DbSyncAdapter.CreateChangesTable(message.Schema.Tables[syncTable.TableName, syncTable.SchemaName], changesSet);
while (dataReader.Read())
{
// Create a row from dataReader
var row = CreateSyncRowFromReader(dataReader, changesSetTable);
// Add the row to the changes set
changesSetTable.Rows.Add(row);
// Set the correct state to be applied
if (row.RowState == DataRowState.Deleted)
{
tableChangesSelected.Deletes++;
}
else if (row.RowState == DataRowState.Modified)
{
tableChangesSelected.Upserts++;
}
// calculate row size if in batch mode
if (isBatch)
{
var fieldsSize = ContainerTable.GetRowSizeFromDataRow(row.ToArray());
var finalFieldSize = fieldsSize / 1024d;
if (finalFieldSize > message.BatchSize)
{
throw new RowOverSizedException(finalFieldSize.ToString());
}
// Calculate the new memory size
rowsMemorySize += finalFieldSize;
// Next line if we don't reach the batch size yet.
if (rowsMemorySize <= message.BatchSize)
{
continue;
}
// add changes to batchinfo
await batchInfo.AddChangesAsync(changesSet, batchIndex, false);
// increment batch index
batchIndex++;
// we know the datas are serialized here, so we can flush the set
changesSet.Clear();
// Recreate an empty ContainerSet and a ContainerTable
changesSet = new SyncSet(message.Schema.ScopeName);
changesSetTable = DbSyncAdapter.CreateChangesTable(message.Schema.Tables[syncTable.TableName, syncTable.SchemaName], changesSet);
// Init the row memory size
rowsMemorySize = 0L;
}
}
}
selectIncrementalChangesCommand.Dispose();
context.SyncStage = SyncStage.TableChangesSelected;
if (tableChangesSelected.Deletes > 0 || tableChangesSelected.Upserts > 0)
{
changes.TableChangesSelected.Add(tableChangesSelected);
}
// Event progress & interceptor
context.SyncStage = SyncStage.TableChangesSelected;
var tableChangesSelectedArgs = new TableChangesSelectedArgs(context, tableChangesSelected, connection, transaction);
this.ReportProgress(context, progress, tableChangesSelectedArgs);
await this.InterceptAsync(tableChangesSelectedArgs).ConfigureAwait(false);
}
// We are in batch mode, and we are at the last batchpart info
// Even if we don't have rows inside, we return the changesSet, since it contains at leaset schema
if (changesSet != null && changesSet.HasTables)
{
await batchInfo.AddChangesAsync(changesSet, batchIndex, true).ConfigureAwait(false);
}
// Check the last index as the last batch
batchInfo.EnsureLastBatch();
return(context, batchInfo, changes);
}