public ServerSyncChanges(long remoteClientTimestamp, BatchInfo serverBatchInfo,
DatabaseChangesSelected serverChangesSelected)
{
this.RemoteClientTimestamp = remoteClientTimestamp;
this.ServerBatchInfo = serverBatchInfo;
this.ServerChangesSelected = serverChangesSelected;
}
InternalApplyThenGetChangesAsync(ClientScopeInfo clientScope, SyncContext context, BatchInfo clientBatchInfo, DbConnection connection = default, DbTransaction transaction = default, CancellationToken cancellationToken = default, IProgress <ProgressArgs> progress = null)
{
try
{
if (Provider == null)
{
throw new MissingProviderException(nameof(InternalApplyThenGetChangesAsync));
}
long remoteClientTimestamp = 0L;
DatabaseChangesSelected serverChangesSelected = null;
DatabaseChangesApplied clientChangesApplied = null;
BatchInfo serverBatchInfo = null;
IScopeInfo serverClientScopeInfo = null;
// Create two transactions
// First one to commit changes
// Second one to get changes now that everything is commited
await using (var runner = await this.GetConnectionAsync(context, SyncMode.Writing, SyncStage.ChangesApplying, connection, transaction, cancellationToken, progress).ConfigureAwait(false))
{
//Direction set to Upload
context.SyncWay = SyncWay.Upload;
// Getting server scope assumes we have already created the schema on server
// Scope name is the scope name coming from client
// Since server can have multiples scopes
(context, serverClientScopeInfo) = await this.InternalLoadServerScopeInfoAsync(context, runner.Connection, runner.Transaction, runner.CancellationToken, runner.Progress).ConfigureAwait(false);
// Should we ?
if (serverClientScopeInfo == null || serverClientScopeInfo.Schema == null)
{
throw new MissingRemoteOrchestratorSchemaException();
}
// Deserialiaze schema
var schema = serverClientScopeInfo.Schema;
// Create message containing everything we need to apply on server side
var applyChanges = new MessageApplyChanges(Guid.Empty, clientScope.Id, false, clientScope.LastServerSyncTimestamp, schema, this.Options.ConflictResolutionPolicy,
this.Options.DisableConstraintsOnApplyChanges, this.Options.CleanMetadatas, this.Options.CleanFolder, false, clientBatchInfo);
// Call provider to apply changes
(context, clientChangesApplied) = await this.InternalApplyChangesAsync(serverClientScopeInfo, context, applyChanges, runner.Connection, runner.Transaction, runner.CancellationToken, runner.Progress).ConfigureAwait(false);
await this.InterceptAsync(new TransactionCommitArgs(context, runner.Connection, runner.Transaction), runner.Progress, runner.CancellationToken).ConfigureAwait(false);
// commit first transaction
await runner.CommitAsync().ConfigureAwait(false);
}
await using (var runner = await this.GetConnectionAsync(context, SyncMode.Reading, SyncStage.ChangesSelecting, connection, transaction, cancellationToken, progress).ConfigureAwait(false))
{
context.ProgressPercentage = 0.55;
//Direction set to Download
context.SyncWay = SyncWay.Download;
// JUST Before get changes, get the timestamp, to be sure to
// get rows inserted / updated elsewhere since the sync is not over
(context, remoteClientTimestamp) = await this.InternalGetLocalTimestampAsync(context, runner.Connection, runner.Transaction, runner.CancellationToken, runner.Progress);
// Get if we need to get all rows from the datasource
var fromScratch = clientScope.IsNewScope || context.SyncType == SyncType.Reinitialize || context.SyncType == SyncType.ReinitializeWithUpload;
// When we get the chnages from server, we create the batches if it's requested by the client
// the batch decision comes from batchsize from client
(context, serverBatchInfo, serverChangesSelected) =
await this.InternalGetChangesAsync(serverClientScopeInfo, context, fromScratch, clientScope.LastServerSyncTimestamp, remoteClientTimestamp, clientScope.Id,
this.Provider.SupportsMultipleActiveResultSets,
this.Options.BatchDirectory, null, runner.Connection, runner.Transaction, runner.CancellationToken, runner.Progress).ConfigureAwait(false);
if (runner.CancellationToken.IsCancellationRequested)
{
runner.CancellationToken.ThrowIfCancellationRequested();
}
// generate the new scope item
this.CompleteTime = DateTime.UtcNow;
var scopeHistory = new ServerHistoryScopeInfo
{
Id = clientScope.Id,
Name = clientScope.Name,
LastSyncTimestamp = remoteClientTimestamp,
LastSync = this.CompleteTime,
LastSyncDuration = this.CompleteTime.Value.Subtract(context.StartTime).Ticks,
};
// Write scopes locally
await this.InternalSaveServerHistoryScopeAsync(scopeHistory, context, runner.Connection, runner.Transaction, runner.CancellationToken, runner.Progress).ConfigureAwait(false);
// Commit second transaction for getting changes
await this.InterceptAsync(new TransactionCommitArgs(context, runner.Connection, runner.Transaction), runner.Progress, runner.CancellationToken).ConfigureAwait(false);
await runner.CommitAsync().ConfigureAwait(false);
}
var serverSyncChanges = new ServerSyncChanges(remoteClientTimestamp, serverBatchInfo, serverChangesSelected);
return(context, serverSyncChanges, clientChangesApplied, this.Options.ConflictResolutionPolicy);
}
catch (Exception ex)
{
throw GetSyncError(context, ex);
}
}
/// <summary>
/// Gets a batch of changes to synchronize when given batch size,
/// destination knowledge, and change data retriever parameters.
/// </summary>
/// <returns>A DbSyncContext object that will be used to retrieve the modified data.</returns>
internal virtual async Task <(SyncContext, BatchInfo, DatabaseChangesSelected)> InternalGetChangesAsync(
SyncContext context, MessageGetChangesBatch message,
DbConnection connection, DbTransaction transaction,
CancellationToken cancellationToken, IProgress <ProgressArgs> progress)
{
// batch info containing changes
BatchInfo batchInfo;
// Statistics about changes that are selected
DatabaseChangesSelected changesSelected;
if (context.SyncWay == SyncWay.Upload && context.SyncType == SyncType.Reinitialize)
{
(batchInfo, changesSelected) = await this.InternalGetEmptyChangesAsync(message).ConfigureAwait(false);
return(context, batchInfo, changesSelected);
}
// Call interceptor
await this.InterceptAsync(new DatabaseChangesSelectingArgs(context, message, connection, transaction), cancellationToken).ConfigureAwait(false);
// create local directory
if (message.BatchSize > 0 && !string.IsNullOrEmpty(message.BatchDirectory) && !Directory.Exists(message.BatchDirectory))
{
Directory.CreateDirectory(message.BatchDirectory);
}
changesSelected = new DatabaseChangesSelected();
// numbers of batch files generated
var batchIndex = 0;
// Check if we are in batch mode
var isBatch = message.BatchSize > 0;
// Create a batch info in memory (if !isBatch) or serialized on disk (if isBatch)
// batchinfo generate a schema clone with scope columns if needed
batchInfo = new BatchInfo(!isBatch, message.Schema, message.BatchDirectory);
// Clean SyncSet, we will add only tables we need in the batch info
var changesSet = new SyncSet();
var cptSyncTable = 0;
var currentProgress = context.ProgressPercentage;
foreach (var syncTable in message.Schema.Tables)
{
// tmp count of table for report progress pct
cptSyncTable++;
// Only table schema is replicated, no datas are applied
if (syncTable.SyncDirection == SyncDirection.None)
{
continue;
}
// if we are in upload stage, so check if table is not download only
if (context.SyncWay == SyncWay.Upload && syncTable.SyncDirection == SyncDirection.DownloadOnly)
{
continue;
}
// if we are in download stage, so check if table is not download only
if (context.SyncWay == SyncWay.Download && syncTable.SyncDirection == SyncDirection.UploadOnly)
{
continue;
}
// Get Command
var selectIncrementalChangesCommand = await this.GetSelectChangesCommandAsync(context, syncTable, message.Setup, message.IsNew, connection, transaction);
// Set parameters
this.SetSelectChangesCommonParameters(context, syncTable, message.ExcludingScopeId, message.IsNew, message.LastTimestamp, selectIncrementalChangesCommand);
// launch interceptor if any
var args = new TableChangesSelectingArgs(context, syncTable, selectIncrementalChangesCommand, connection, transaction);
await this.InterceptAsync(args, cancellationToken).ConfigureAwait(false);
if (!args.Cancel && args.Command != null)
{
// Statistics
var tableChangesSelected = new TableChangesSelected(syncTable.TableName, syncTable.SchemaName);
// Create a chnages table with scope columns
var changesSetTable = DbSyncAdapter.CreateChangesTable(message.Schema.Tables[syncTable.TableName, syncTable.SchemaName], changesSet);
// Get the reader
using var dataReader = await args.Command.ExecuteReaderAsync().ConfigureAwait(false);
// memory size total
double rowsMemorySize = 0L;
while (dataReader.Read())
{
// Create a row from dataReader
var row = CreateSyncRowFromReader(dataReader, changesSetTable);
// Add the row to the changes set
changesSetTable.Rows.Add(row);
// Set the correct state to be applied
if (row.RowState == DataRowState.Deleted)
{
tableChangesSelected.Deletes++;
}
else if (row.RowState == DataRowState.Modified)
{
tableChangesSelected.Upserts++;
}
// calculate row size if in batch mode
if (isBatch)
{
var fieldsSize = ContainerTable.GetRowSizeFromDataRow(row.ToArray());
var finalFieldSize = fieldsSize / 1024d;
if (finalFieldSize > message.BatchSize)
{
throw new RowOverSizedException(finalFieldSize.ToString());
}
// Calculate the new memory size
rowsMemorySize += finalFieldSize;
// Next line if we don't reach the batch size yet.
if (rowsMemorySize <= message.BatchSize)
{
continue;
}
// Check interceptor
var batchTableChangesSelectedArgs = new TableChangesSelectedArgs(context, changesSetTable, tableChangesSelected, connection, transaction);
await this.InterceptAsync(batchTableChangesSelectedArgs, cancellationToken).ConfigureAwait(false);
// add changes to batchinfo
await batchInfo.AddChangesAsync(changesSet, batchIndex, false, message.SerializerFactory, this).ConfigureAwait(false);
// increment batch index
batchIndex++;
// we know the datas are serialized here, so we can flush the set
changesSet.Clear();
// Recreate an empty ContainerSet and a ContainerTable
changesSet = new SyncSet();
changesSetTable = DbSyncAdapter.CreateChangesTable(message.Schema.Tables[syncTable.TableName, syncTable.SchemaName], changesSet);
// Init the row memory size
rowsMemorySize = 0L;
}
}
dataReader.Close();
// We don't report progress if no table changes is empty, to limit verbosity
if (tableChangesSelected.Deletes > 0 || tableChangesSelected.Upserts > 0)
{
changesSelected.TableChangesSelected.Add(tableChangesSelected);
}
// even if no rows raise the interceptor
var tableChangesSelectedArgs = new TableChangesSelectedArgs(context, changesSetTable, tableChangesSelected, connection, transaction);
await this.InterceptAsync(tableChangesSelectedArgs, cancellationToken).ConfigureAwait(false);
context.ProgressPercentage = currentProgress + (cptSyncTable * 0.2d / message.Schema.Tables.Count);
// only raise report progress if we have something
if (tableChangesSelectedArgs.TableChangesSelected.TotalChanges > 0)
{
this.ReportProgress(context, progress, tableChangesSelectedArgs);
}
}
}
// We are in batch mode, and we are at the last batchpart info
// Even if we don't have rows inside, we return the changesSet, since it contains at least schema
if (changesSet != null && changesSet.HasTables && changesSet.HasRows)
{
await batchInfo.AddChangesAsync(changesSet, batchIndex, true, message.SerializerFactory, this).ConfigureAwait(false);
}
//Set the total rows count contained in the batch info
batchInfo.RowsCount = changesSelected.TotalChangesSelected;
// Check the last index as the last batch
batchInfo.EnsureLastBatch();
// Raise database changes selected
if (changesSelected.TotalChangesSelected > 0 || changesSelected.TotalChangesSelectedDeletes > 0 || changesSelected.TotalChangesSelectedUpdates > 0)
{
var databaseChangesSelectedArgs = new DatabaseChangesSelectedArgs(context, message.LastTimestamp, batchInfo, changesSelected, connection);
this.ReportProgress(context, progress, databaseChangesSelectedArgs);
await this.InterceptAsync(databaseChangesSelectedArgs, cancellationToken).ConfigureAwait(false);
}
return(context, batchInfo, changesSelected);
}
/// <summary>
/// Gets changes rows count estimation,
/// </summary>
internal virtual async Task <(SyncContext, DatabaseChangesSelected)> InternalGetEstimatedChangesCountAsync(
SyncContext context, MessageGetChangesBatch message,
DbConnection connection, DbTransaction transaction,
CancellationToken cancellationToken, IProgress <ProgressArgs> progress)
{
// Call interceptor
await this.InterceptAsync(new DatabaseChangesSelectingArgs(context, message, connection, transaction), cancellationToken).ConfigureAwait(false);
// Create stats object to store changes count
var changes = new DatabaseChangesSelected();
if (context.SyncWay == SyncWay.Upload && context.SyncType == SyncType.Reinitialize)
{
return(context, changes);
}
foreach (var syncTable in message.Schema.Tables)
{
// Only table schema is replicated, no datas are applied
if (syncTable.SyncDirection == SyncDirection.None)
{
continue;
}
// if we are in upload stage, so check if table is not download only
if (context.SyncWay == SyncWay.Upload && syncTable.SyncDirection == SyncDirection.DownloadOnly)
{
continue;
}
// if we are in download stage, so check if table is not download only
if (context.SyncWay == SyncWay.Download && syncTable.SyncDirection == SyncDirection.UploadOnly)
{
continue;
}
// Get Command
var command = await this.GetSelectChangesCommandAsync(context, syncTable, message.Setup, message.IsNew, connection, transaction);
// Set parameters
this.SetSelectChangesCommonParameters(context, syncTable, message.ExcludingScopeId, message.IsNew, message.LastTimestamp, command);
// launch interceptor if any
var args = new TableChangesSelectingArgs(context, syncTable, command, connection, transaction);
await this.InterceptAsync(args, cancellationToken).ConfigureAwait(false);
if (args.Cancel || args.Command == null)
{
continue;
}
// Statistics
var tableChangesSelected = new TableChangesSelected(syncTable.TableName, syncTable.SchemaName);
// Get the reader
using var dataReader = await args.Command.ExecuteReaderAsync().ConfigureAwait(false);
while (dataReader.Read())
{
bool isTombstone = false;
for (var i = 0; i < dataReader.FieldCount; i++)
{
if (dataReader.GetName(i) == "sync_row_is_tombstone")
{
isTombstone = Convert.ToInt64(dataReader.GetValue(i)) > 0;
break;
}
}
// Set the correct state to be applied
if (isTombstone)
{
tableChangesSelected.Deletes++;
}
else
{
tableChangesSelected.Upserts++;
}
}
dataReader.Close();
// Check interceptor
var changesArgs = new TableChangesSelectedArgs(context, null, tableChangesSelected, connection, transaction);
await this.InterceptAsync(changesArgs, cancellationToken).ConfigureAwait(false);
if (tableChangesSelected.Deletes > 0 || tableChangesSelected.Upserts > 0)
{
changes.TableChangesSelected.Add(tableChangesSelected);
}
}
// Raise database changes selected
var databaseChangesSelectedArgs = new DatabaseChangesSelectedArgs(context, message.LastTimestamp, null, changes, connection);
this.ReportProgress(context, progress, databaseChangesSelectedArgs);
await this.InterceptAsync(databaseChangesSelectedArgs, cancellationToken).ConfigureAwait(false);
return(context, changes);
}
/// <summary>
/// Launch a synchronization with the specified mode
/// </summary>
public async Task <SyncContext> SynchronizeAsync(SyncType syncType, CancellationToken cancellationToken, IProgress <ProgressArgs> progress = null)
{
// Context, used to back and forth data between servers
var context = new SyncContext(Guid.NewGuid())
{
// set start time
StartTime = DateTime.Now,
// if any parameters, set in context
Parameters = this.Parameters,
// set sync type (Normal, Reinitialize, ReinitializeWithUpload)
SyncType = syncType
};
this.SessionState = SyncSessionState.Synchronizing;
this.SessionStateChanged?.Invoke(this, this.SessionState);
ScopeInfo localScopeInfo = null,
serverScopeInfo = null,
localScopeReferenceInfo = null,
scope = null;
var fromId = Guid.Empty;
var lastSyncTS = 0L;
var isNew = true;
try
{
if (cancellationToken.IsCancellationRequested)
{
cancellationToken.ThrowIfCancellationRequested();
}
// Setting the cancellation token
this.LocalProvider.SetCancellationToken(cancellationToken);
this.RemoteProvider.SetCancellationToken(cancellationToken);
// Setting progress
this.LocalProvider.SetProgress(progress);
// ----------------------------------------
// 0) Begin Session / Get the Configuration from remote provider
// If the configuration object is provided by the client, the server will be updated with it.
// ----------------------------------------
(context, this.LocalProvider.Configuration) = await this.RemoteProvider.BeginSessionAsync(context,
new MessageBeginSession { Configuration = this.LocalProvider.Configuration });
if (cancellationToken.IsCancellationRequested)
{
cancellationToken.ThrowIfCancellationRequested();
}
// Locally, nothing really special. Eventually, editing the config object
(context, this.LocalProvider.Configuration) = await this.LocalProvider.BeginSessionAsync(context,
new MessageBeginSession { Configuration = this.LocalProvider.Configuration });
if (cancellationToken.IsCancellationRequested)
{
cancellationToken.ThrowIfCancellationRequested();
}
// ----------------------------------------
// 1) Read scope info
// ----------------------------------------
// get the scope from local provider
List <ScopeInfo> localScopes;
List <ScopeInfo> serverScopes;
(context, localScopes) = await this.LocalProvider.EnsureScopesAsync(context,
new MessageEnsureScopes
{
ScopeInfoTableName = this.LocalProvider.Configuration.ScopeInfoTableName,
ScopeName = this.LocalProvider.Configuration.ScopeName,
SerializationFormat = this.LocalProvider.Configuration.SerializationFormat
});
if (localScopes.Count != 1)
{
throw new Exception("On Local provider, we should have only one scope info");
}
if (cancellationToken.IsCancellationRequested)
{
cancellationToken.ThrowIfCancellationRequested();
}
localScopeInfo = localScopes[0];
(context, serverScopes) = await this.RemoteProvider.EnsureScopesAsync(context,
new MessageEnsureScopes
{
ScopeInfoTableName = this.LocalProvider.Configuration.ScopeInfoTableName,
ScopeName = this.LocalProvider.Configuration.ScopeName,
ClientReferenceId = localScopeInfo.Id,
SerializationFormat = this.LocalProvider.Configuration.SerializationFormat
});
if (serverScopes.Count != 2)
{
throw new Exception("On Remote provider, we should have two scopes (one for server and one for client side)");
}
if (cancellationToken.IsCancellationRequested)
{
cancellationToken.ThrowIfCancellationRequested();
}
serverScopeInfo = serverScopes.First(s => s.Id != localScopeInfo.Id);
localScopeReferenceInfo = serverScopes.First(s => s.Id == localScopeInfo.Id);
// ----------------------------------------
// 2) Build Configuration Object
// ----------------------------------------
// Get Schema from remote provider
(context, this.LocalProvider.Configuration.Schema) = await this.RemoteProvider.EnsureSchemaAsync(context,
new MessageEnsureSchema
{
Schema = this.LocalProvider.Configuration.Schema,
SerializationFormat = this.LocalProvider.Configuration.SerializationFormat
});
if (cancellationToken.IsCancellationRequested)
{
cancellationToken.ThrowIfCancellationRequested();
}
// Apply on local Provider
(context, this.LocalProvider.Configuration.Schema) = await this.LocalProvider.EnsureSchemaAsync(context,
new MessageEnsureSchema
{
Schema = this.LocalProvider.Configuration.Schema,
SerializationFormat = this.LocalProvider.Configuration.SerializationFormat
});
if (cancellationToken.IsCancellationRequested)
{
cancellationToken.ThrowIfCancellationRequested();
}
// ----------------------------------------
// 3) Ensure databases are ready
// ----------------------------------------
// Server should have already the schema
context = await this.RemoteProvider.EnsureDatabaseAsync(context,
new MessageEnsureDatabase
{
ScopeInfo = serverScopeInfo,
Schema = this.LocalProvider.Configuration.Schema,
Filters = this.LocalProvider.Configuration.Filters,
SerializationFormat = this.LocalProvider.Configuration.SerializationFormat
});
if (cancellationToken.IsCancellationRequested)
{
cancellationToken.ThrowIfCancellationRequested();
}
// Client could have, or not, the tables
context = await this.LocalProvider.EnsureDatabaseAsync(context,
new MessageEnsureDatabase
{
ScopeInfo = localScopeInfo,
Schema = this.LocalProvider.Configuration.Schema,
Filters = this.LocalProvider.Configuration.Filters,
SerializationFormat = this.LocalProvider.Configuration.SerializationFormat
});
if (cancellationToken.IsCancellationRequested)
{
cancellationToken.ThrowIfCancellationRequested();
}
// ----------------------------------------
// 5) Get changes and apply them
// ----------------------------------------
BatchInfo clientBatchInfo;
BatchInfo serverBatchInfo;
DatabaseChangesSelected clientChangesSelected = null;
DatabaseChangesSelected serverChangesSelected = null;
DatabaseChangesApplied clientChangesApplied = null;
DatabaseChangesApplied serverChangesApplied = null;
// those timestamps will be registered as the "timestamp just before launch the sync"
long serverTimestamp, clientTimestamp;
if (cancellationToken.IsCancellationRequested)
{
cancellationToken.ThrowIfCancellationRequested();
}
// Apply on the Server Side
// Since we are on the server,
// we need to check the server client timestamp (not the client timestamp which is completely different)
var serverPolicy = this.LocalProvider.Configuration.ConflictResolutionPolicy;
var clientPolicy = serverPolicy == ConflictResolutionPolicy.ServerWins ? ConflictResolutionPolicy.ClientWins : ConflictResolutionPolicy.ServerWins;
// We get from local provider all rows not last updated from the server
fromId = serverScopeInfo.Id;
// lastSyncTS : get lines inserted / updated / deteleted after the last sync commited
lastSyncTS = localScopeInfo.LastSyncTimestamp;
// isNew : If isNew, lasttimestamp is not correct, so grab all
isNew = localScopeInfo.IsNewScope;
//Direction set to Upload
context.SyncWay = SyncWay.Upload;
// JUST before the whole process, get the timestamp, to be sure to
// get rows inserted / updated elsewhere since the sync is not over
(context, clientTimestamp) = await this.LocalProvider.GetLocalTimestampAsync(context,
new MessageTimestamp
{
ScopeInfoTableName = this.LocalProvider.Configuration.ScopeInfoTableName,
SerializationFormat = this.LocalProvider.Configuration.SerializationFormat
});
if (cancellationToken.IsCancellationRequested)
{
cancellationToken.ThrowIfCancellationRequested();
}
scope = new ScopeInfo {
Id = fromId, IsNewScope = isNew, Timestamp = lastSyncTS
};
(context, clientBatchInfo, clientChangesSelected) =
await this.LocalProvider.GetChangeBatchAsync(context,
new MessageGetChangesBatch
{
ScopeInfo = scope,
Schema = this.LocalProvider.Configuration.Schema,
Policy = clientPolicy,
Filters = this.LocalProvider.Configuration.Filters,
SerializationFormat = this.LocalProvider.Configuration.SerializationFormat
});
if (cancellationToken.IsCancellationRequested)
{
cancellationToken.ThrowIfCancellationRequested();
}
// fromId : When applying rows, make sure it's identified as applied by this client scope
fromId = localScopeInfo.Id;
// lastSyncTS : apply lines only if thye are not modified since last client sync
lastSyncTS = localScopeReferenceInfo.LastSyncTimestamp;
// isNew : not needed
isNew = false;
scope = new ScopeInfo {
Id = fromId, IsNewScope = isNew, Timestamp = lastSyncTS
};
(context, serverChangesApplied) =
await this.RemoteProvider.ApplyChangesAsync(context,
new MessageApplyChanges
{
FromScope = scope,
Schema = this.LocalProvider.Configuration.Schema,
Policy = serverPolicy,
ScopeInfoTableName = this.LocalProvider.Configuration.ScopeInfoTableName,
Changes = clientBatchInfo,
SerializationFormat = this.LocalProvider.Configuration.SerializationFormat
});
// if ConflictResolutionPolicy.ClientWins or Handler set to Client wins
// Conflict occurs here and server loose.
// Conflicts count should be temp saved because applychanges on client side won't raise any conflicts (and so property Context.TotalSyncConflicts will be reset to 0)
var conflictsOnRemoteCount = context.TotalSyncConflicts;
if (cancellationToken.IsCancellationRequested)
{
cancellationToken.ThrowIfCancellationRequested();
}
// Get changes from server
// fromId : Make sure we don't select lines on server that has been already updated by the client
fromId = localScopeInfo.Id;
// lastSyncTS : apply lines only if thye are not modified since last client sync
lastSyncTS = localScopeReferenceInfo.LastSyncTimestamp;
// isNew : make sure we take all lines if it's the first time we get
isNew = localScopeReferenceInfo.IsNewScope;
scope = new ScopeInfo {
Id = fromId, IsNewScope = isNew, Timestamp = lastSyncTS
};
//Direction set to Download
context.SyncWay = SyncWay.Download;
// JUST Before get changes, get the timestamp, to be sure to
// get rows inserted / updated elsewhere since the sync is not over
(context, serverTimestamp) = await this.RemoteProvider.GetLocalTimestampAsync(context,
new MessageTimestamp
{
ScopeInfoTableName = this.LocalProvider.Configuration.ScopeInfoTableName,
SerializationFormat = this.LocalProvider.Configuration.SerializationFormat
});
if (cancellationToken.IsCancellationRequested)
{
cancellationToken.ThrowIfCancellationRequested();
}
(context, serverBatchInfo, serverChangesSelected) =
await this.RemoteProvider.GetChangeBatchAsync(context,
new MessageGetChangesBatch
{
ScopeInfo = scope,
Schema = this.LocalProvider.Configuration.Schema,
Policy = serverPolicy,
Filters = this.LocalProvider.Configuration.Filters,
SerializationFormat = this.LocalProvider.Configuration.SerializationFormat
});
if (cancellationToken.IsCancellationRequested)
{
cancellationToken.ThrowIfCancellationRequested();
}
// Apply local changes
// fromId : When applying rows, make sure it's identified as applied by this server scope
fromId = serverScopeInfo.Id;
// lastSyncTS : apply lines only if they are not modified since last client sync
lastSyncTS = localScopeInfo.LastSyncTimestamp;
// isNew : if IsNew, don't apply deleted rows from server
isNew = localScopeInfo.IsNewScope;
scope = new ScopeInfo {
Id = fromId, IsNewScope = isNew, Timestamp = lastSyncTS
};
(context, clientChangesApplied) =
await this.LocalProvider.ApplyChangesAsync(context,
new MessageApplyChanges
{
FromScope = scope,
Schema = this.LocalProvider.Configuration.Schema,
Policy = clientPolicy,
ScopeInfoTableName = this.LocalProvider.Configuration.ScopeInfoTableName,
Changes = serverBatchInfo,
SerializationFormat = this.LocalProvider.Configuration.SerializationFormat
});
context.TotalChangesDownloaded = clientChangesApplied.TotalAppliedChanges;
context.TotalChangesUploaded = clientChangesSelected.TotalChangesSelected;
context.TotalSyncErrors = clientChangesApplied.TotalAppliedChangesFailed;
context.CompleteTime = DateTime.Now;
serverScopeInfo.IsNewScope = false;
localScopeReferenceInfo.IsNewScope = false;
localScopeInfo.IsNewScope = false;
serverScopeInfo.LastSync = context.CompleteTime;
localScopeReferenceInfo.LastSync = context.CompleteTime;
localScopeInfo.LastSync = context.CompleteTime;
serverScopeInfo.LastSyncTimestamp = serverTimestamp;
localScopeReferenceInfo.LastSyncTimestamp = serverTimestamp;
localScopeInfo.LastSyncTimestamp = clientTimestamp;
var duration = context.CompleteTime.Subtract(context.StartTime);
serverScopeInfo.LastSyncDuration = duration.Ticks;
localScopeReferenceInfo.LastSyncDuration = duration.Ticks;
localScopeInfo.LastSyncDuration = duration.Ticks;
serverScopeInfo.IsLocal = true;
localScopeReferenceInfo.IsLocal = false;
context = await this.RemoteProvider.WriteScopesAsync(context,
new MessageWriteScopes
{
ScopeInfoTableName = this.LocalProvider.Configuration.ScopeInfoTableName,
Scopes = new List <ScopeInfo> {
serverScopeInfo, localScopeReferenceInfo
},
SerializationFormat = this.LocalProvider.Configuration.SerializationFormat
});
serverScopeInfo.IsLocal = false;
localScopeInfo.IsLocal = true;
if (cancellationToken.IsCancellationRequested)
{
cancellationToken.ThrowIfCancellationRequested();
}
context = await this.LocalProvider.WriteScopesAsync(context,
new MessageWriteScopes
{
ScopeInfoTableName = this.LocalProvider.Configuration.ScopeInfoTableName,
Scopes = new List <ScopeInfo> {
localScopeInfo, serverScopeInfo
},
SerializationFormat = this.LocalProvider.Configuration.SerializationFormat
});
if (cancellationToken.IsCancellationRequested)
{
cancellationToken.ThrowIfCancellationRequested();
}
}
catch (SyncException se)
{
Console.WriteLine($"Sync Exception: {se.Message}. Type:{se.Type}.");
throw;
}
catch (Exception ex)
{
Console.WriteLine($"Unknwon Exception: {ex.Message}.");
throw new SyncException(ex, SyncStage.None);
}
finally
{
// End the current session
context = await this.RemoteProvider.EndSessionAsync(context);
context = await this.LocalProvider.EndSessionAsync(context);
this.SessionState = SyncSessionState.Ready;
this.SessionStateChanged?.Invoke(this, this.SessionState);
}
return(context);
}
public ClientSyncChanges(long clientTimestamp, BatchInfo clientBatchInfo, DatabaseChangesSelected clientChangesSelected)
{
this.ClientTimestamp = clientTimestamp;
this.ClientBatchInfo = clientBatchInfo;
this.ClientChangesSelected = clientChangesSelected;
}
/// <summary>
/// Gets changes rows count estimation,
/// </summary>
public virtual async Task <(SyncContext, DatabaseChangesSelected)> GetEstimatedChangesCountAsync(
SyncContext context, MessageGetChangesBatch message,
DbConnection connection, DbTransaction transaction,
CancellationToken cancellationToken, IProgress <ProgressArgs> progress)
{
this.Orchestrator.logger.LogDebug(SyncEventsId.GetChanges, message);
// Create stats object to store changes count
var changes = new DatabaseChangesSelected();
if (context.SyncWay == SyncWay.Upload && context.SyncType == SyncType.Reinitialize)
{
return(context, changes);
}
foreach (var syncTable in message.Schema.Tables)
{
this.Orchestrator.logger.LogDebug(SyncEventsId.GetChanges, syncTable);
// if we are in upload stage, so check if table is not download only
if (context.SyncWay == SyncWay.Upload && syncTable.SyncDirection == SyncDirection.DownloadOnly)
{
continue;
}
// if we are in download stage, so check if table is not download only
if (context.SyncWay == SyncWay.Download && syncTable.SyncDirection == SyncDirection.UploadOnly)
{
continue;
}
var tableBuilder = this.GetTableBuilder(syncTable, message.Setup);
var syncAdapter = tableBuilder.CreateSyncAdapter();
// launch interceptor if any
await this.Orchestrator.InterceptAsync(new TableChangesSelectingArgs(context, syncTable, connection, transaction), cancellationToken).ConfigureAwait(false);
// Get Command
var selectIncrementalChangesCommand = await this.GetSelectChangesCommandAsync(context, syncAdapter, syncTable, message.IsNew, connection, transaction);
// Set parameters
this.SetSelectChangesCommonParameters(context, syncTable, message.ExcludingScopeId, message.IsNew, message.LastTimestamp, selectIncrementalChangesCommand);
// log
this.Orchestrator.logger.LogDebug(SyncEventsId.GetChanges, selectIncrementalChangesCommand);
// Statistics
var tableChangesSelected = new TableChangesSelected(syncTable.TableName, syncTable.SchemaName);
// Get the reader
using (var dataReader = await selectIncrementalChangesCommand.ExecuteReaderAsync().ConfigureAwait(false))
{
while (dataReader.Read())
{
bool isTombstone = false;
for (var i = 0; i < dataReader.FieldCount; i++)
{
if (dataReader.GetName(i) == "sync_row_is_tombstone")
{
isTombstone = Convert.ToInt64(dataReader.GetValue(i)) > 0;
break;
}
}
// Set the correct state to be applied
if (isTombstone)
{
tableChangesSelected.Deletes++;
}
else
{
tableChangesSelected.Upserts++;
}
}
}
if (tableChangesSelected.Deletes > 0 || tableChangesSelected.Upserts > 0)
{
changes.TableChangesSelected.Add(tableChangesSelected);
}
}
return(context, changes);
}
InternalApplyChangesAsync(ClientScopeInfo clientScopeInfo, SyncContext context, BatchInfo serverBatchInfo,
long clientTimestamp, long remoteClientTimestamp, ConflictResolutionPolicy policy, bool snapshotApplied, DatabaseChangesSelected allChangesSelected,
DbConnection connection = default, DbTransaction transaction = default, CancellationToken cancellationToken = default, IProgress <ProgressArgs> progress = null)
{
try
{
// lastSyncTS : apply lines only if they are not modified since last client sync
var lastTimestamp = clientScopeInfo.LastSyncTimestamp;
// isNew : if IsNew, don't apply deleted rows from server
var isNew = clientScopeInfo.IsNewScope;
// We are in downloading mode
// Create the message containing everything needed to apply changes
var applyChanges = new MessageApplyChanges(clientScopeInfo.Id, Guid.Empty, isNew, lastTimestamp, clientScopeInfo.Schema, policy,
this.Options.DisableConstraintsOnApplyChanges, this.Options.CleanMetadatas, this.Options.CleanFolder, snapshotApplied,
serverBatchInfo);
DatabaseChangesApplied clientChangesApplied;
await using var runner = await this.GetConnectionAsync(context, SyncMode.Writing, SyncStage.ChangesApplying, connection, transaction, cancellationToken, progress).ConfigureAwait(false);
context.SyncWay = SyncWay.Download;
// Call apply changes on provider
(context, clientChangesApplied) = await this.InternalApplyChangesAsync(clientScopeInfo, context, applyChanges, runner.Connection, runner.Transaction, cancellationToken, progress).ConfigureAwait(false);
if (cancellationToken.IsCancellationRequested)
{
cancellationToken.ThrowIfCancellationRequested();
}
// check if we need to delete metadatas
if (this.Options.CleanMetadatas && clientChangesApplied.TotalAppliedChanges > 0 && lastTimestamp.HasValue)
{
List <ClientScopeInfo> allScopes;
(context, allScopes) = await this.InternalLoadAllClientScopesInfoAsync(context, runner.Connection, runner.Transaction, runner.CancellationToken, runner.Progress).ConfigureAwait(false);
if (allScopes.Count > 0)
{
// Get the min value from LastSyncTimestamp from all scopes
var minLastTimeStamp = allScopes.Min(scope => scope.LastSyncTimestamp.HasValue ? scope.LastSyncTimestamp.Value : Int64.MaxValue);
minLastTimeStamp = minLastTimeStamp > lastTimestamp.Value ? lastTimestamp.Value : minLastTimeStamp;
(context, _) = await this.InternalDeleteMetadatasAsync(allScopes, context, minLastTimeStamp, runner.Connection, runner.Transaction, cancellationToken, progress).ConfigureAwait(false);
}
}
// now the sync is complete, remember the time
this.CompleteTime = DateTime.UtcNow;
// generate the new scope item
clientScopeInfo.IsNewScope = false;
clientScopeInfo.LastSync = this.CompleteTime;
clientScopeInfo.LastSyncTimestamp = clientTimestamp;
clientScopeInfo.LastServerSyncTimestamp = remoteClientTimestamp;
clientScopeInfo.LastSyncDuration = this.CompleteTime.Value.Subtract(context.StartTime).Ticks;
// Write scopes locally
(context, clientScopeInfo) = await this.InternalSaveClientScopeInfoAsync(clientScopeInfo, context, runner.Connection, runner.Transaction, cancellationToken, progress).ConfigureAwait(false);
await runner.CommitAsync().ConfigureAwait(false);
return(context, clientChangesApplied, clientScopeInfo);
}
catch (Exception ex)
{
throw GetSyncError(context, ex);
}
}
public DatabaseChangesSelectedArgs(SyncContext context, long?timestamp, BatchInfo clientBatchInfo, DatabaseChangesSelected changesSelected, DbConnection connection = null, DbTransaction transaction = null)
: base(context, connection, transaction)
{
this.Timestamp = timestamp;
this.BatchInfo = clientBatchInfo;
this.ChangesSelected = changesSelected;
}
/// <summary>
/// Gets a batch of changes to synchronize when given batch size,
/// destination knowledge, and change data retriever parameters.
/// </summary>
/// <returns>A DbSyncContext object that will be used to retrieve the modified data.</returns>
internal virtual async Task <(SyncContext, BatchInfo, DatabaseChangesSelected)> InternalGetChangesAsync(
IScopeInfo scopeInfo, SyncContext context, bool isNew, long?fromLastTimestamp, long?toNewTimestamp, Guid?excludingScopeId,
bool supportsMultiActiveResultSets, string batchRootDirectory, string batchDirectoryName,
DbConnection connection, DbTransaction transaction,
CancellationToken cancellationToken, IProgress <ProgressArgs> progress)
{
// batch info containing changes
BatchInfo batchInfo;
// Statistics about changes that are selected
DatabaseChangesSelected changesSelected;
context.SyncStage = SyncStage.ChangesSelecting;
if (context.SyncWay == SyncWay.Upload && context.SyncType == SyncType.Reinitialize)
{
(batchInfo, changesSelected) = await this.InternalGetEmptyChangesAsync(scopeInfo, batchRootDirectory).ConfigureAwait(false);
return(context, batchInfo, changesSelected);
}
// create local directory
if (!string.IsNullOrEmpty(batchRootDirectory) && !Directory.Exists(batchRootDirectory))
{
Directory.CreateDirectory(batchRootDirectory);
}
changesSelected = new DatabaseChangesSelected();
// Create a batch
// batchinfo generate a schema clone with scope columns if needed
batchInfo = new BatchInfo(scopeInfo.Schema, batchRootDirectory, batchDirectoryName);
batchInfo.TryRemoveDirectory();
batchInfo.CreateDirectory();
// Call interceptor
var databaseChangesSelectingArgs = new DatabaseChangesSelectingArgs(context, batchInfo.GetDirectoryFullPath(), this.Options.BatchSize, isNew,
fromLastTimestamp, toNewTimestamp, connection, transaction);
await this.InterceptAsync(databaseChangesSelectingArgs, progress, cancellationToken).ConfigureAwait(false);
var cptSyncTable = 0;
var currentProgress = context.ProgressPercentage;
var schemaTables = scopeInfo.Schema.Tables.SortByDependencies(tab => tab.GetRelations().Select(r => r.GetParentTable()));
var lstAllBatchPartInfos = new ConcurrentBag <BatchPartInfo>();
var lstTableChangesSelected = new ConcurrentBag <TableChangesSelected>();
var threadNumberLimits = supportsMultiActiveResultSets ? 16 : 1;
if (supportsMultiActiveResultSets)
{
await schemaTables.ForEachAsync(async syncTable =>
{
if (cancellationToken.IsCancellationRequested)
{
return;
}
// tmp count of table for report progress pct
cptSyncTable++;
List <BatchPartInfo> syncTableBatchPartInfos;
TableChangesSelected tableChangesSelected;
(context, syncTableBatchPartInfos, tableChangesSelected) = await InternalReadSyncTableChangesAsync(
scopeInfo, context, excludingScopeId, syncTable, batchInfo, isNew, fromLastTimestamp, connection, transaction, cancellationToken, progress).ConfigureAwait(false);
if (syncTableBatchPartInfos == null)
{
return;
}
// We don't report progress if no table changes is empty, to limit verbosity
if (tableChangesSelected != null && (tableChangesSelected.Deletes > 0 || tableChangesSelected.Upserts > 0))
{
lstTableChangesSelected.Add(tableChangesSelected);
}
// Add sync table bpi to all bpi
syncTableBatchPartInfos.ForEach(bpi => lstAllBatchPartInfos.Add(bpi));
context.ProgressPercentage = currentProgress + (cptSyncTable * 0.2d / scopeInfo.Schema.Tables.Count);
}, threadNumberLimits);
}
else
{
foreach (var syncTable in schemaTables)
{
if (cancellationToken.IsCancellationRequested)
{
continue;
}
// tmp count of table for report progress pct
cptSyncTable++;
List <BatchPartInfo> syncTableBatchPartInfos;
TableChangesSelected tableChangesSelected;
(context, syncTableBatchPartInfos, tableChangesSelected) = await InternalReadSyncTableChangesAsync(
scopeInfo, context, excludingScopeId, syncTable, batchInfo, isNew, fromLastTimestamp, connection, transaction, cancellationToken, progress).ConfigureAwait(false);
if (syncTableBatchPartInfos == null)
{
continue;
}
// We don't report progress if no table changes is empty, to limit verbosity
if (tableChangesSelected != null && (tableChangesSelected.Deletes > 0 || tableChangesSelected.Upserts > 0))
{
lstTableChangesSelected.Add(tableChangesSelected);
}
// Add sync table bpi to all bpi
syncTableBatchPartInfos.ForEach(bpi => lstAllBatchPartInfos.Add(bpi));
context.ProgressPercentage = currentProgress + (cptSyncTable * 0.2d / scopeInfo.Schema.Tables.Count);
}
}
while (!lstTableChangesSelected.IsEmpty)
{
if (lstTableChangesSelected.TryTake(out var tableChangesSelected))
{
changesSelected.TableChangesSelected.Add(tableChangesSelected);
}
}
// delete all empty batchparts (empty tables)
foreach (var bpi in lstAllBatchPartInfos.Where(bpi => bpi.RowsCount <= 0))
{
File.Delete(Path.Combine(batchInfo.GetDirectoryFullPath(), bpi.FileName));
}
// Generate a good index order to be compliant with previous versions
var tmpLstBatchPartInfos = new List <BatchPartInfo>();
foreach (var table in schemaTables)
{
// get all bpi where count > 0 and ordered by index
foreach (var bpi in lstAllBatchPartInfos.Where(bpi => bpi.RowsCount > 0 && bpi.Tables[0].EqualsByName(new BatchPartTableInfo(table.TableName, table.SchemaName))).OrderBy(bpi => bpi.Index).ToArray())
{
batchInfo.BatchPartsInfo.Add(bpi);
batchInfo.RowsCount += bpi.RowsCount;
tmpLstBatchPartInfos.Add(bpi);
}
}
var newBatchIndex = 0;
foreach (var bpi in tmpLstBatchPartInfos)
{
bpi.Index = newBatchIndex;
newBatchIndex++;
bpi.IsLastBatch = newBatchIndex == tmpLstBatchPartInfos.Count;
}
//Set the total rows count contained in the batch info
batchInfo.EnsureLastBatch();
if (batchInfo.RowsCount <= 0)
{
var cleanFolder = await this.InternalCanCleanFolderAsync(scopeInfo.Name, context.Parameters, batchInfo, cancellationToken).ConfigureAwait(false);
batchInfo.Clear(cleanFolder);
}
var databaseChangesSelectedArgs = new DatabaseChangesSelectedArgs(context, fromLastTimestamp, toNewTimestamp, batchInfo, changesSelected, connection);
await this.InterceptAsync(databaseChangesSelectedArgs, progress, cancellationToken).ConfigureAwait(false);
return(context, batchInfo, changesSelected);
}
/// <summary>
/// Enumerate all internal changes, no batch mode
/// </summary>
internal async Task <(BatchInfo, DatabaseChangesSelected)> EnumerateChangesInBatchesInternalAsync
(SyncContext context, ScopeInfo scopeInfo, int downloadBatchSizeInKB, DmSet configTables, string batchDirectory, ConflictResolutionPolicy policy, ICollection <FilterClause> filters)
{
DmTable dmTable = null;
// memory size total
double memorySizeFromDmRows = 0L;
var batchIndex = 0;
// this batch info won't be in memory, it will be be batched
var batchInfo = new BatchInfo(false, batchDirectory);
// directory where all files will be stored
batchInfo.GenerateNewDirectoryName();
// Create stats object to store changes count
var changes = new DatabaseChangesSelected();
using (var connection = this.CreateConnection())
{
try
{
// Open the connection
await connection.OpenAsync();
using (var transaction = connection.BeginTransaction())
{
// create the in memory changes set
var changesSet = new DmSet(configTables.DmSetName);
foreach (var tableDescription in configTables.Tables)
{
// if we are in upload stage, so check if table is not download only
if (context.SyncWay == SyncWay.Upload && tableDescription.SyncDirection == SyncDirection.DownloadOnly)
{
continue;
}
// if we are in download stage, so check if table is not download only
if (context.SyncWay == SyncWay.Download && tableDescription.SyncDirection == SyncDirection.UploadOnly)
{
continue;
}
var builder = this.GetDatabaseBuilder(tableDescription);
var syncAdapter = builder.CreateSyncAdapter(connection, transaction);
// raise before event
context.SyncStage = SyncStage.TableChangesSelecting;
var tableChangesSelectingArgs = new TableChangesSelectingArgs(context, tableDescription.TableName, connection, transaction);
// launc interceptor if any
await this.InterceptAsync(tableChangesSelectingArgs);
// Get Command
DbCommand selectIncrementalChangesCommand;
DbCommandType dbCommandType;
if (this.CanBeServerProvider && context.Parameters != null && context.Parameters.Count > 0 && filters != null && filters.Count > 0)
{
var tableFilters = filters
.Where(f => f.TableName.Equals(tableDescription.TableName, StringComparison.InvariantCultureIgnoreCase));
if (tableFilters != null && tableFilters.Count() > 0)
{
dbCommandType = DbCommandType.SelectChangesWitFilters;
selectIncrementalChangesCommand = syncAdapter.GetCommand(dbCommandType, tableFilters);
if (selectIncrementalChangesCommand == null)
{
throw new Exception("Missing command 'SelectIncrementalChangesCommand' ");
}
syncAdapter.SetCommandParameters(dbCommandType, selectIncrementalChangesCommand, tableFilters);
}
else
{
dbCommandType = DbCommandType.SelectChanges;
selectIncrementalChangesCommand = syncAdapter.GetCommand(dbCommandType);
if (selectIncrementalChangesCommand == null)
{
throw new Exception("Missing command 'SelectIncrementalChangesCommand' ");
}
syncAdapter.SetCommandParameters(dbCommandType, selectIncrementalChangesCommand);
}
}
else
{
dbCommandType = DbCommandType.SelectChanges;
selectIncrementalChangesCommand = syncAdapter.GetCommand(dbCommandType);
if (selectIncrementalChangesCommand == null)
{
throw new Exception("Missing command 'SelectIncrementalChangesCommand' ");
}
syncAdapter.SetCommandParameters(dbCommandType, selectIncrementalChangesCommand);
}
dmTable = this.BuildChangesTable(tableDescription.TableName, configTables);
try
{
// Set commons parameters
SetSelectChangesCommonParameters(context, scopeInfo, selectIncrementalChangesCommand);
// Set filter parameters if any
// Only on server side
if (this.CanBeServerProvider && context.Parameters != null && context.Parameters.Count > 0 && filters != null && filters.Count > 0)
{
var filterTable = filters.Where(f => f.TableName.Equals(tableDescription.TableName, StringComparison.InvariantCultureIgnoreCase)).ToList();
if (filterTable != null && filterTable.Count > 0)
{
foreach (var filter in filterTable)
{
var parameter = context.Parameters.FirstOrDefault(p => p.ColumnName.Equals(filter.ColumnName, StringComparison.InvariantCultureIgnoreCase) && p.TableName.Equals(filter.TableName, StringComparison.InvariantCultureIgnoreCase));
if (parameter != null)
{
DbManager.SetParameterValue(selectIncrementalChangesCommand, parameter.ColumnName, parameter.Value);
}
}
}
}
this.AddTrackingColumns <int>(dmTable, "sync_row_is_tombstone");
// Statistics
var tableChangesSelected = new TableChangesSelected
{
TableName = tableDescription.TableName
};
changes.TableChangesSelected.Add(tableChangesSelected);
// Get the reader
using (var dataReader = selectIncrementalChangesCommand.ExecuteReader())
{
while (dataReader.Read())
{
var dmRow = this.CreateRowFromReader(dataReader, dmTable);
var state = DmRowState.Unchanged;
state = this.GetStateFromDmRow(dmRow, scopeInfo);
// If the row is not deleted inserted or modified, go next
if (state != DmRowState.Deleted && state != DmRowState.Modified && state != DmRowState.Added)
{
continue;
}
var fieldsSize = DmTableSurrogate.GetRowSizeFromDataRow(dmRow);
var dmRowSize = fieldsSize / 1024d;
if (dmRowSize > downloadBatchSizeInKB)
{
var exc = $"Row is too big ({dmRowSize} kb.) for the current Configuration.DownloadBatchSizeInKB ({downloadBatchSizeInKB} kb.) Aborting Sync...";
throw new Exception(exc);
}
// Calculate the new memory size
memorySizeFromDmRows = memorySizeFromDmRows + dmRowSize;
// add row
dmTable.Rows.Add(dmRow);
tableChangesSelected.TotalChanges++;
// acceptchanges before modifying
dmRow.AcceptChanges();
// Set the correct state to be applied
if (state == DmRowState.Deleted)
{
dmRow.Delete();
tableChangesSelected.Deletes++;
}
else if (state == DmRowState.Added)
{
dmRow.SetAdded();
tableChangesSelected.Inserts++;
}
else if (state == DmRowState.Modified)
{
dmRow.SetModified();
tableChangesSelected.Updates++;
}
// We exceed the memorySize, so we can add it to a batch
if (memorySizeFromDmRows > downloadBatchSizeInKB)
{
// Since we dont need this column anymore, remove it
this.RemoveTrackingColumns(dmTable, "sync_row_is_tombstone");
changesSet.Tables.Add(dmTable);
// generate the batch part info
batchInfo.GenerateBatchInfo(batchIndex, changesSet);
// increment batch index
batchIndex++;
changesSet.Clear();
// Recreate an empty DmSet, then a dmTable clone
changesSet = new DmSet(configTables.DmSetName);
dmTable = dmTable.Clone();
this.AddTrackingColumns <int>(dmTable, "sync_row_is_tombstone");
// Init the row memory size
memorySizeFromDmRows = 0L;
// SyncProgress & interceptor
context.SyncStage = SyncStage.TableChangesSelected;
var loopTableChangesSelectedArgs = new TableChangesSelectedArgs(context, tableChangesSelected, connection, transaction);
this.ReportProgress(context, loopTableChangesSelectedArgs);
await this.InterceptAsync(loopTableChangesSelectedArgs);
}
}
// Since we dont need this column anymore, remove it
this.RemoveTrackingColumns(dmTable, "sync_row_is_tombstone");
context.SyncStage = SyncStage.TableChangesSelected;
changesSet.Tables.Add(dmTable);
// Init the row memory size
memorySizeFromDmRows = 0L;
// Event progress & interceptor
context.SyncStage = SyncStage.TableChangesSelected;
var tableChangesSelectedArgs = new TableChangesSelectedArgs(context, tableChangesSelected, connection, transaction);
this.ReportProgress(context, tableChangesSelectedArgs);
await this.InterceptAsync(tableChangesSelectedArgs);
}
}
catch (Exception)
{
throw;
}
finally
{
}
}
// We are in batch mode, and we are at the last batchpart info
if (changesSet != null && changesSet.HasTables && changesSet.HasChanges())
{
var batchPartInfo = batchInfo.GenerateBatchInfo(batchIndex, changesSet);
if (batchPartInfo != null)
{
batchPartInfo.IsLastBatch = true;
}
}
transaction.Commit();
}
}
catch (Exception)
{
throw;
}
finally
{
if (connection != null && connection.State == ConnectionState.Open)
{
connection.Close();
}
}
}
return(batchInfo, changes);
}
/// <summary>
/// Enumerate all internal changes, no batch mode
/// </summary>
internal async Task <(BatchInfo, DatabaseChangesSelected)> EnumerateChangesInternalAsync(
SyncContext context, ScopeInfo scopeInfo, DmSet configTables, string batchDirectory, ConflictResolutionPolicy policy, ICollection <FilterClause> filters)
{
// create the in memory changes set
var changesSet = new DmSet(SyncConfiguration.DMSET_NAME);
// Create the batch info, in memory
// No need to geneate a directory name, since we are in memory
var batchInfo = new BatchInfo(true, batchDirectory);
using (var connection = this.CreateConnection())
{
// Open the connection
await connection.OpenAsync();
using (var transaction = connection.BeginTransaction())
{
try
{
// changes that will be returned as selected changes
var changes = new DatabaseChangesSelected();
foreach (var tableDescription in configTables.Tables)
{
// if we are in upload stage, so check if table is not download only
if (context.SyncWay == SyncWay.Upload && tableDescription.SyncDirection == SyncDirection.DownloadOnly)
{
continue;
}
// if we are in download stage, so check if table is not download only
if (context.SyncWay == SyncWay.Download && tableDescription.SyncDirection == SyncDirection.UploadOnly)
{
continue;
}
var builder = this.GetDatabaseBuilder(tableDescription);
var syncAdapter = builder.CreateSyncAdapter(connection, transaction);
// raise before event
context.SyncStage = SyncStage.TableChangesSelecting;
// launch any interceptor
await this.InterceptAsync(new TableChangesSelectingArgs(context, tableDescription.TableName, connection, transaction));
// selected changes for the current table
var tableSelectedChanges = new TableChangesSelected
{
TableName = tableDescription.TableName
};
// Get Command
DbCommand selectIncrementalChangesCommand;
DbCommandType dbCommandType;
if (this.CanBeServerProvider && context.Parameters != null && context.Parameters.Count > 0 && filters != null && filters.Count > 0)
{
var tableFilters = filters
.Where(f => f.TableName.Equals(tableDescription.TableName, StringComparison.InvariantCultureIgnoreCase));
if (tableFilters != null && tableFilters.Count() > 0)
{
dbCommandType = DbCommandType.SelectChangesWitFilters;
selectIncrementalChangesCommand = syncAdapter.GetCommand(dbCommandType, tableFilters);
if (selectIncrementalChangesCommand == null)
{
throw new Exception("Missing command 'SelectIncrementalChangesCommand'");
}
syncAdapter.SetCommandParameters(dbCommandType, selectIncrementalChangesCommand, tableFilters);
}
else
{
dbCommandType = DbCommandType.SelectChanges;
selectIncrementalChangesCommand = syncAdapter.GetCommand(dbCommandType);
if (selectIncrementalChangesCommand == null)
{
throw new Exception("Missing command 'SelectIncrementalChangesCommand'");
}
syncAdapter.SetCommandParameters(dbCommandType, selectIncrementalChangesCommand);
}
}
else
{
dbCommandType = DbCommandType.SelectChanges;
selectIncrementalChangesCommand = syncAdapter.GetCommand(dbCommandType);
if (selectIncrementalChangesCommand == null)
{
throw new Exception("Missing command 'SelectIncrementalChangesCommand'");
}
syncAdapter.SetCommandParameters(dbCommandType, selectIncrementalChangesCommand);
}
// Get a clone of the table with tracking columns
var dmTableChanges = this.BuildChangesTable(tableDescription.TableName, configTables);
SetSelectChangesCommonParameters(context, scopeInfo, selectIncrementalChangesCommand);
// Set filter parameters if any
if (this.CanBeServerProvider && context.Parameters != null && context.Parameters.Count > 0 && filters != null && filters.Count > 0)
{
var tableFilters = filters
.Where(f => f.TableName.Equals(tableDescription.TableName, StringComparison.InvariantCultureIgnoreCase)).ToList();
if (tableFilters != null && tableFilters.Count > 0)
{
foreach (var filter in tableFilters)
{
var parameter = context.Parameters.FirstOrDefault(p => p.ColumnName.Equals(filter.ColumnName, StringComparison.InvariantCultureIgnoreCase) && p.TableName.Equals(filter.TableName, StringComparison.InvariantCultureIgnoreCase));
if (parameter != null)
{
DbManager.SetParameterValue(selectIncrementalChangesCommand, parameter.ColumnName, parameter.Value);
}
}
}
}
this.AddTrackingColumns <int>(dmTableChanges, "sync_row_is_tombstone");
// Get the reader
using (var dataReader = selectIncrementalChangesCommand.ExecuteReader())
{
while (dataReader.Read())
{
var dataRow = this.CreateRowFromReader(dataReader, dmTableChanges);
//DmRow dataRow = dmTableChanges.NewRow();
// assuming the row is not inserted / modified
var state = DmRowState.Unchanged;
// get if the current row is inserted, modified, deleted
state = this.GetStateFromDmRow(dataRow, scopeInfo);
if (state != DmRowState.Deleted && state != DmRowState.Modified && state != DmRowState.Added)
{
continue;
}
// add row
dmTableChanges.Rows.Add(dataRow);
// acceptchanges before modifying
dataRow.AcceptChanges();
tableSelectedChanges.TotalChanges++;
// Set the correct state to be applied
if (state == DmRowState.Deleted)
{
dataRow.Delete();
tableSelectedChanges.Deletes++;
}
else if (state == DmRowState.Added)
{
dataRow.SetAdded();
tableSelectedChanges.Inserts++;
}
else if (state == DmRowState.Modified)
{
dataRow.SetModified();
tableSelectedChanges.Updates++;
}
}
// Since we dont need this column anymore, remove it
this.RemoveTrackingColumns(dmTableChanges, "sync_row_is_tombstone");
// add it to the DmSet
changesSet.Tables.Add(dmTableChanges);
}
// add the stats to global stats
changes.TableChangesSelected.Add(tableSelectedChanges);
// Progress & Interceptor
context.SyncStage = SyncStage.TableChangesSelected;
var args = new TableChangesSelectedArgs(context, tableSelectedChanges, connection, transaction);
this.ReportProgress(context, args);
await this.InterceptAsync(args);
}
transaction.Commit();
// generate the batchpartinfo
batchInfo.GenerateBatchInfo(0, changesSet);
// Create a new in-memory batch info with an the changes DmSet
return(batchInfo, changes);
}
catch (Exception)
{
throw;
}
finally
{
if (connection != null && connection.State == ConnectionState.Open)
{
connection.Close();
}
}
}
}
}
/// <summary>
/// Gets a batch of changes to synchronize when given batch size,
/// destination knowledge, and change data retriever parameters.
/// </summary>
/// <returns>A DbSyncContext object that will be used to retrieve the modified data.</returns>
public virtual async Task <(SyncContext, BatchInfo, DatabaseChangesSelected)> GetChangeBatchAsync(
SyncContext context, MessageGetChangesBatch message,
DbConnection connection, DbTransaction transaction,
CancellationToken cancellationToken, IProgress <ProgressArgs> progress = null)
{
// batch info containing changes
BatchInfo batchInfo;
// Statistics about changes that are selected
DatabaseChangesSelected changesSelected;
if (context.SyncWay == SyncWay.Upload && context.SyncType == SyncType.Reinitialize)
{
(batchInfo, changesSelected) = this.GetEmptyChanges(message);
return(context, batchInfo, changesSelected);
}
// Check if the provider is not outdated
var isOutdated = this.IsRemoteOutdated();
// Get a chance to make the sync even if it's outdated
if (isOutdated)
{
var outdatedArgs = new OutdatedArgs(context, null, null);
// Interceptor
await this.InterceptAsync(outdatedArgs).ConfigureAwait(false);
if (outdatedArgs.Action != OutdatedAction.Rollback)
{
context.SyncType = outdatedArgs.Action == OutdatedAction.Reinitialize ? SyncType.Reinitialize : SyncType.ReinitializeWithUpload;
}
if (outdatedArgs.Action == OutdatedAction.Rollback)
{
throw new OutOfDateException();
}
}
// create local directory
if (message.BatchSize > 0 && !string.IsNullOrEmpty(message.BatchDirectory) && !Directory.Exists(message.BatchDirectory))
{
Directory.CreateDirectory(message.BatchDirectory);
}
// numbers of batch files generated
var batchIndex = 0;
// Check if we are in batch mode
var isBatch = message.BatchSize > 0;
// Create stats object to store changes count
var changes = new DatabaseChangesSelected();
// create the in memory changes set
var changesSet = new SyncSet(message.Schema.ScopeName);
// Create a Schema set without readonly columns, attached to memory changes
foreach (var table in message.Schema.Tables)
{
DbSyncAdapter.CreateChangesTable(message.Schema.Tables[table.TableName, table.SchemaName], changesSet);
}
// Create a batch info in memory (if !isBatch) or serialized on disk (if isBatch)
// batchinfo generate a schema clone with scope columns if needed
batchInfo = new BatchInfo(!isBatch, changesSet, message.BatchDirectory);
// Clear tables, we will add only the ones we need in the batch info
changesSet.Clear();
foreach (var syncTable in message.Schema.Tables)
{
// if we are in upload stage, so check if table is not download only
if (context.SyncWay == SyncWay.Upload && syncTable.SyncDirection == SyncDirection.DownloadOnly)
{
continue;
}
// if we are in download stage, so check if table is not download only
if (context.SyncWay == SyncWay.Download && syncTable.SyncDirection == SyncDirection.UploadOnly)
{
continue;
}
var tableBuilder = this.GetTableBuilder(syncTable);
var syncAdapter = tableBuilder.CreateSyncAdapter(connection, transaction);
// raise before event
context.SyncStage = SyncStage.TableChangesSelecting;
var tableChangesSelectingArgs = new TableChangesSelectingArgs(context, syncTable.TableName, connection, transaction);
// launch interceptor if any
await this.InterceptAsync(tableChangesSelectingArgs).ConfigureAwait(false);
// Get Command
var selectIncrementalChangesCommand = this.GetSelectChangesCommand(context, syncAdapter, syncTable, message.IsNew);
// Set parameters
this.SetSelectChangesCommonParameters(context, syncTable, message.ExcludingScopeId, message.IsNew, message.LastTimestamp, selectIncrementalChangesCommand);
// Statistics
var tableChangesSelected = new TableChangesSelected(syncTable.TableName);
// Get the reader
using (var dataReader = selectIncrementalChangesCommand.ExecuteReader())
{
// memory size total
double rowsMemorySize = 0L;
// Create a chnages table with scope columns
var changesSetTable = DbSyncAdapter.CreateChangesTable(message.Schema.Tables[syncTable.TableName, syncTable.SchemaName], changesSet);
while (dataReader.Read())
{
// Create a row from dataReader
var row = CreateSyncRowFromReader(dataReader, changesSetTable);
// Add the row to the changes set
changesSetTable.Rows.Add(row);
// Set the correct state to be applied
if (row.RowState == DataRowState.Deleted)
{
tableChangesSelected.Deletes++;
}
else if (row.RowState == DataRowState.Modified)
{
tableChangesSelected.Upserts++;
}
// calculate row size if in batch mode
if (isBatch)
{
var fieldsSize = ContainerTable.GetRowSizeFromDataRow(row.ToArray());
var finalFieldSize = fieldsSize / 1024d;
if (finalFieldSize > message.BatchSize)
{
throw new RowOverSizedException(finalFieldSize.ToString());
}
// Calculate the new memory size
rowsMemorySize += finalFieldSize;
// Next line if we don't reach the batch size yet.
if (rowsMemorySize <= message.BatchSize)
{
continue;
}
// add changes to batchinfo
batchInfo.AddChanges(changesSet, batchIndex, false);
// increment batch index
batchIndex++;
// we know the datas are serialized here, so we can flush the set
changesSet.Clear();
// Recreate an empty ContainerSet and a ContainerTable
changesSet = new SyncSet(message.Schema.ScopeName);
changesSetTable = DbSyncAdapter.CreateChangesTable(message.Schema.Tables[syncTable.TableName, syncTable.SchemaName], changesSet);
// Init the row memory size
rowsMemorySize = 0L;
}
}
}
selectIncrementalChangesCommand.Dispose();
context.SyncStage = SyncStage.TableChangesSelected;
if (tableChangesSelected.Deletes > 0 || tableChangesSelected.Upserts > 0)
{
changes.TableChangesSelected.Add(tableChangesSelected);
}
// Event progress & interceptor
context.SyncStage = SyncStage.TableChangesSelected;
var tableChangesSelectedArgs = new TableChangesSelectedArgs(context, tableChangesSelected, connection, transaction);
this.ReportProgress(context, progress, tableChangesSelectedArgs);
await this.InterceptAsync(tableChangesSelectedArgs).ConfigureAwait(false);
}
// We are in batch mode, and we are at the last batchpart info
// Even if we don't have rows inside, we return the changesSet, since it contains at leaset schema
if (changesSet != null && changesSet.HasTables)
{
batchInfo.AddChanges(changesSet, batchIndex, true);
}
// Check the last index as the last batch
batchInfo.EnsureLastBatch();
return(context, batchInfo, changes);
}
ApplyThenGetChangesAsync(ScopeInfo clientScope, BatchInfo clientBatchInfo,
CancellationToken cancellationToken = default, IProgress <ProgressArgs> progress = null)
{
if (!this.StartTime.HasValue)
{
this.StartTime = DateTime.UtcNow;
}
// Get context or create a new one
var ctx = this.GetContext();
long remoteClientTimestamp = 0L;
DatabaseChangesSelected serverChangesSelected = null;
DatabaseChangesApplied clientChangesApplied = null;
BatchInfo serverBatchInfo = null;
SyncSet schema = null;
using var connection = this.Provider.CreateConnection();
try
{
ctx.SyncStage = SyncStage.ChangesApplying;
//Direction set to Upload
ctx.SyncWay = SyncWay.Upload;
// Open connection
await this.OpenConnectionAsync(connection, cancellationToken).ConfigureAwait(false);
DbTransaction transaction;
// Create two transactions
// First one to commit changes
// Second one to get changes now that everything is commited
using (transaction = connection.BeginTransaction())
{
await this.InterceptAsync(new TransactionOpenedArgs(ctx, connection, transaction), cancellationToken).ConfigureAwait(false);
// Maybe here, get the schema from server, issue from client scope name
// Maybe then compare the schema version from client scope with schema version issued from server
// Maybe if different, raise an error ?
// Get scope if exists
// Getting server scope assumes we have already created the schema on server
var scopeBuilder = this.GetScopeBuilder(this.Options.ScopeInfoTableName);
var serverScopeInfo = await this.InternalGetScopeAsync <ServerScopeInfo>(ctx, DbScopeType.Server, clientScope.Name, scopeBuilder, connection, transaction, cancellationToken, progress).ConfigureAwait(false);
// Should we ?
if (serverScopeInfo.Schema == null)
{
throw new MissingRemoteOrchestratorSchemaException();
}
// Deserialiaze schema
schema = serverScopeInfo.Schema;
// Create message containing everything we need to apply on server side
var applyChanges = new MessageApplyChanges(Guid.Empty, clientScope.Id, false, clientScope.LastServerSyncTimestamp, schema, this.Setup, this.Options.ConflictResolutionPolicy,
this.Options.DisableConstraintsOnApplyChanges, this.Options.UseBulkOperations, this.Options.CleanMetadatas, this.Options.CleanFolder, false, clientBatchInfo, this.Options.SerializerFactory);
// Call provider to apply changes
(ctx, clientChangesApplied) = await this.InternalApplyChangesAsync(ctx, applyChanges, connection, transaction, cancellationToken, progress).ConfigureAwait(false);
await this.InterceptAsync(new TransactionCommitArgs(ctx, connection, transaction), cancellationToken).ConfigureAwait(false);
// commit first transaction
transaction.Commit();
}
ctx.SyncStage = SyncStage.ChangesSelecting;
ctx.ProgressPercentage = 0.55;
using (transaction = connection.BeginTransaction())
{
await this.InterceptAsync(new TransactionOpenedArgs(ctx, connection, transaction), cancellationToken).ConfigureAwait(false);
//Direction set to Download
ctx.SyncWay = SyncWay.Download;
// JUST Before get changes, get the timestamp, to be sure to
// get rows inserted / updated elsewhere since the sync is not over
remoteClientTimestamp = await this.InternalGetLocalTimestampAsync(ctx, connection, transaction, cancellationToken, progress);
// Get if we need to get all rows from the datasource
var fromScratch = clientScope.IsNewScope || ctx.SyncType == SyncType.Reinitialize || ctx.SyncType == SyncType.ReinitializeWithUpload;
var message = new MessageGetChangesBatch(clientScope.Id, Guid.Empty, fromScratch, clientScope.LastServerSyncTimestamp, schema, this.Setup, this.Options.BatchSize, this.Options.BatchDirectory, this.Options.SerializerFactory);
// Call interceptor
await this.InterceptAsync(new DatabaseChangesSelectingArgs(ctx, message, connection, transaction), cancellationToken).ConfigureAwait(false);
// When we get the chnages from server, we create the batches if it's requested by the client
// the batch decision comes from batchsize from client
(ctx, serverBatchInfo, serverChangesSelected) =
await this.InternalGetChangesAsync(ctx, message, connection, transaction, cancellationToken, progress).ConfigureAwait(false);
if (cancellationToken.IsCancellationRequested)
{
cancellationToken.ThrowIfCancellationRequested();
}
// generate the new scope item
this.CompleteTime = DateTime.UtcNow;
var scopeHistory = new ServerHistoryScopeInfo
{
Id = clientScope.Id,
Name = clientScope.Name,
LastSyncTimestamp = remoteClientTimestamp,
LastSync = this.CompleteTime,
LastSyncDuration = this.CompleteTime.Value.Subtract(this.StartTime.Value).Ticks,
};
// Write scopes locally
var scopeBuilder = this.GetScopeBuilder(this.Options.ScopeInfoTableName);
await this.InternalSaveScopeAsync(ctx, DbScopeType.ServerHistory, scopeHistory, scopeBuilder, connection, transaction, cancellationToken, progress).ConfigureAwait(false);
// Commit second transaction for getting changes
await this.InterceptAsync(new TransactionCommitArgs(ctx, connection, transaction), cancellationToken).ConfigureAwait(false);
transaction.Commit();
}
// Event progress & interceptor
await this.CloseConnectionAsync(connection, cancellationToken).ConfigureAwait(false);
}
catch (Exception ex)
{
RaiseError(ex);
}
finally
{
await this.CloseConnectionAsync(connection, cancellationToken).ConfigureAwait(false);
}
return(remoteClientTimestamp, serverBatchInfo, this.Options.ConflictResolutionPolicy, clientChangesApplied, serverChangesSelected);
}
GetEstimatedChangesCountAsync(ScopeInfo localScopeInfo = null, CancellationToken cancellationToken = default, IProgress <ProgressArgs> progress = null)
{
if (!this.StartTime.HasValue)
{
this.StartTime = DateTime.UtcNow;
}
// Output
long clientTimestamp = 0L;
DatabaseChangesSelected clientChangesSelected = null;
// Get context or create a new one
var ctx = this.GetContext();
using (var connection = this.Provider.CreateConnection())
{
try
{
ctx.SyncStage = SyncStage.ChangesSelecting;
// Open connection
await this.OpenConnectionAsync(connection, cancellationToken).ConfigureAwait(false);
using (var transaction = connection.BeginTransaction())
{
await this.InterceptAsync(new TransactionOpenedArgs(ctx, connection, transaction), cancellationToken).ConfigureAwait(false);
// Get local scope, if not provided
if (localScopeInfo == null)
{
ctx = await this.Provider.EnsureClientScopeAsync(ctx, this.Options.ScopeInfoTableName, connection, transaction, cancellationToken, progress).ConfigureAwait(false);
(ctx, localScopeInfo) = await this.Provider.GetClientScopeAsync(ctx, this.Options.ScopeInfoTableName, this.ScopeName, connection, transaction, cancellationToken, progress).ConfigureAwait(false);
}
// If no schema in the client scope. Maybe the client scope table does not exists, or we never get the schema from server
if (localScopeInfo.Schema == null)
{
throw new MissingLocalOrchestratorSchemaException();
}
this.logger.LogInformation(SyncEventsId.GetClientScope, localScopeInfo);
// On local, we don't want to chase rows from "others"
// We just want our local rows, so we dont exclude any remote scope id, by setting scope id to NULL
Guid?remoteScopeId = null;
// lastSyncTS : get lines inserted / updated / deteleted after the last sync commited
var lastSyncTS = localScopeInfo.LastSyncTimestamp;
// isNew : If isNew, lasttimestamp is not correct, so grab all
var isNew = localScopeInfo.IsNewScope;
//Direction set to Upload
ctx.SyncWay = SyncWay.Upload;
// JUST before the whole process, get the timestamp, to be sure to
// get rows inserted / updated elsewhere since the sync is not over
clientTimestamp = await this.Provider.GetLocalTimestampAsync(ctx, connection, transaction, cancellationToken, progress);
// Creating the message
// Since it's an estimated count, we don't need to create batches, so we hard code the batchsize to 0
var message = new MessageGetChangesBatch(remoteScopeId, localScopeInfo.Id, isNew, lastSyncTS, localScopeInfo.Schema, this.Setup, 0, this.Options.BatchDirectory);
// Call interceptor
await this.InterceptAsync(new DatabaseChangesSelectingArgs(ctx, message, connection, transaction), cancellationToken).ConfigureAwait(false);
this.logger.LogDebug(SyncEventsId.GetChanges, message);
// Locally, if we are new, no need to get changes
if (isNew)
{
clientChangesSelected = new DatabaseChangesSelected();
}
else
{
(ctx, clientChangesSelected) = await this.Provider.GetEstimatedChangesCountAsync(ctx, message, connection, transaction, cancellationToken, progress).ConfigureAwait(false);
}
await this.InterceptAsync(new TransactionCommitArgs(ctx, connection, transaction), cancellationToken).ConfigureAwait(false);
transaction.Commit();
}
// Event progress & interceptor
ctx.SyncStage = SyncStage.ChangesSelected;
await this.CloseConnectionAsync(connection, cancellationToken).ConfigureAwait(false);
}
catch (Exception ex)
{
RaiseError(ex);
}
finally
{
if (connection != null && connection.State != ConnectionState.Closed)
{
connection.Close();
}
}
this.logger.LogInformation(SyncEventsId.GetChanges, new { ClientTimestamp = clientTimestamp, ClientChangesSelected = clientChangesSelected });
return(clientTimestamp, clientChangesSelected);
}
}
InternalGetSnapshotAsync(ServerScopeInfo serverScopeInfo, SyncContext context, DbConnection connection = default, DbTransaction transaction = default, CancellationToken cancellationToken = default, IProgress <ProgressArgs> progress = null)
{
try
{
await using var runner = await this.GetConnectionAsync(context, SyncMode.Reading, SyncStage.ScopeLoading, connection, transaction, cancellationToken, progress).ConfigureAwait(false);
// Get context or create a new one
var changesSelected = new DatabaseChangesSelected();
BatchInfo serverBatchInfo = null;
if (string.IsNullOrEmpty(this.Options.SnapshotsDirectory))
{
return(context, 0, null, changesSelected);
}
//Direction set to Download
context.SyncWay = SyncWay.Download;
if (cancellationToken.IsCancellationRequested)
{
cancellationToken.ThrowIfCancellationRequested();
}
// Get Schema from remote provider if no schema passed from args
if (serverScopeInfo.Schema == null)
{
(context, serverScopeInfo) = await this.InternalGetServerScopeInfoAsync(context, serverScopeInfo.Setup, false, runner.Connection, runner.Transaction, runner.CancellationToken, runner.Progress).ConfigureAwait(false);
}
// When we get the changes from server, we create the batches if it's requested by the client
// the batch decision comes from batchsize from client
var(rootDirectory, nameDirectory) = await this.InternalGetSnapshotDirectoryPathAsync(serverScopeInfo.Name, context.Parameters, runner.CancellationToken, runner.Progress).ConfigureAwait(false);
if (!string.IsNullOrEmpty(rootDirectory))
{
var directoryFullPath = Path.Combine(rootDirectory, nameDirectory);
// if no snapshot present, just return null value.
if (Directory.Exists(directoryFullPath))
{
// Serialize on disk.
var jsonConverter = new Serialization.JsonConverter <BatchInfo>();
var summaryFileName = Path.Combine(directoryFullPath, "summary.json");
using (var fs = new FileStream(summaryFileName, FileMode.Open, FileAccess.Read))
{
serverBatchInfo = await jsonConverter.DeserializeAsync(fs).ConfigureAwait(false);
}
// Create the schema changeset
var changesSet = new SyncSet();
// Create a Schema set without readonly columns, attached to memory changes
foreach (var table in serverScopeInfo.Schema.Tables)
{
DbSyncAdapter.CreateChangesTable(serverScopeInfo.Schema.Tables[table.TableName, table.SchemaName], changesSet);
// Get all stats about this table
var bptis = serverBatchInfo.BatchPartsInfo.SelectMany(bpi => bpi.Tables.Where(t =>
{
var sc = SyncGlobalization.DataSourceStringComparison;
var sn = t.SchemaName == null ? string.Empty : t.SchemaName;
var otherSn = table.SchemaName == null ? string.Empty : table.SchemaName;
return(table.TableName.Equals(t.TableName, sc) && sn.Equals(otherSn, sc));
}));
if (bptis != null)
{
// Statistics
var tableChangesSelected = new TableChangesSelected(table.TableName, table.SchemaName)
{
// we are applying a snapshot where it can't have any deletes, obviously
Upserts = bptis.Sum(bpti => bpti.RowsCount)
};
if (tableChangesSelected.Upserts > 0)
{
changesSelected.TableChangesSelected.Add(tableChangesSelected);
}
}
}
serverBatchInfo.SanitizedSchema = changesSet;
}
}
if (serverBatchInfo == null)
{
return(context, 0, null, changesSelected);
}
await runner.CommitAsync().ConfigureAwait(false);
return(context, serverBatchInfo.Timestamp, serverBatchInfo, changesSelected);
}
catch (Exception ex)
{
throw GetSyncError(context, ex);
}
}
InternalApplySnapshotAsync(ClientScopeInfo clientScopeInfo, SyncContext context, BatchInfo serverBatchInfo, long clientTimestamp, long remoteClientTimestamp, DatabaseChangesSelected databaseChangesSelected,
DbConnection connection = default, DbTransaction transaction = default, CancellationToken cancellationToken = default, IProgress <ProgressArgs> progress = null)
{
try
{
if (serverBatchInfo == null)
{
return(context, new DatabaseChangesApplied(), clientScopeInfo);
}
// Get context or create a new one
context.SyncStage = SyncStage.SnapshotApplying;
await this.InterceptAsync(new SnapshotApplyingArgs(context, this.Provider.CreateConnection()), progress, cancellationToken).ConfigureAwait(false);
if (clientScopeInfo.Schema == null)
{
throw new ArgumentNullException(nameof(clientScopeInfo.Schema));
}
// Applying changes and getting the new client scope info
var(syncContext, changesApplied, newClientScopeInfo) = await this.InternalApplyChangesAsync(clientScopeInfo, context, serverBatchInfo,
clientTimestamp, remoteClientTimestamp, ConflictResolutionPolicy.ServerWins, false, databaseChangesSelected, connection, transaction, cancellationToken, progress).ConfigureAwait(false);
await this.InterceptAsync(new SnapshotAppliedArgs(context, changesApplied), progress, cancellationToken).ConfigureAwait(false);
// re-apply scope is new flag
// to be sure we are calling the Initialize method, even for the delta
// in that particular case, we want the delta rows coming from the current scope
newClientScopeInfo.IsNewScope = true;
return(context, changesApplied, newClientScopeInfo);
}
catch (Exception ex)
{
throw GetSyncError(context, ex);
}
}