/// <summary>
/// update configuration object with tables desc from server database
/// </summary>
public virtual async Task <(SyncContext, BatchInfo)> CreateSnapshotAsync(SyncContext context, SyncSet schema, SyncSetup setup,
DbConnection connection, DbTransaction transaction, string snapshotDirectory, int batchSize, long remoteClientTimestamp,
CancellationToken cancellationToken, IProgress <ProgressArgs> progress = null)
{
// create local directory
if (!Directory.Exists(snapshotDirectory))
{
this.Orchestrator.logger.LogDebug(SyncEventsId.CreateDirectory, new { SnapshotDirectory = snapshotDirectory });
Directory.CreateDirectory(snapshotDirectory);
}
// cleansing scope name
var directoryScopeName = new string(context.ScopeName.Where(char.IsLetterOrDigit).ToArray());
var directoryFullPath = Path.Combine(snapshotDirectory, directoryScopeName);
// create local directory with scope inside
if (!Directory.Exists(directoryFullPath))
{
this.Orchestrator.logger.LogDebug(SyncEventsId.CreateDirectory, new { DirectoryFullPath = directoryFullPath });
Directory.CreateDirectory(directoryFullPath);
}
// numbers of batch files generated
var batchIndex = 0;
// create the in memory changes set
var changesSet = new SyncSet();
var sb = new StringBuilder();
var underscore = "";
if (context.Parameters != null)
{
foreach (var p in context.Parameters.OrderBy(p => p.Name))
{
var cleanValue = new string(p.Value.ToString().Where(char.IsLetterOrDigit).ToArray());
var cleanName = new string(p.Name.Where(char.IsLetterOrDigit).ToArray());
sb.Append($"{underscore}{cleanName}_{cleanValue}");
underscore = "_";
}
}
var directoryName = sb.ToString();
directoryName = string.IsNullOrEmpty(directoryName) ? "ALL" : directoryName;
// batchinfo generate a schema clone with scope columns if needed
var batchInfo = new BatchInfo(false, schema, directoryFullPath, directoryName);
// Delete directory if already exists
directoryFullPath = Path.Combine(directoryFullPath, directoryName);
if (Directory.Exists(directoryFullPath))
{
this.Orchestrator.logger.LogDebug(SyncEventsId.DropDirectory, new { DirectoryFullPath = directoryFullPath });
Directory.Delete(directoryFullPath, true);
}
foreach (var syncTable in schema.Tables)
{
var tableBuilder = this.GetTableBuilder(syncTable, setup);
var syncAdapter = tableBuilder.CreateSyncAdapter();
// launch interceptor if any
await this.Orchestrator.InterceptAsync(new TableChangesSelectingArgs(context, syncTable, connection, transaction), cancellationToken).ConfigureAwait(false);
// Get Select initialize changes command
var selectIncrementalChangesCommand = await this.GetSelectChangesCommandAsync(context, syncAdapter, syncTable, true, connection, transaction);
// Set parameters
this.SetSelectChangesCommonParameters(context, syncTable, null, true, 0, selectIncrementalChangesCommand);
// log
this.Orchestrator.logger.LogDebug(SyncEventsId.CreateSnapshot, new
{
SelectChangesCommandText = selectIncrementalChangesCommand.CommandText,
ExcludingScopeId = Guid.Empty,
IsNew = true,
LastTimestamp = 0
});
// Get the reader
using (var dataReader = await selectIncrementalChangesCommand.ExecuteReaderAsync().ConfigureAwait(false))
{
// memory size total
double rowsMemorySize = 0L;
// Create a chnages table with scope columns
var changesSetTable = SyncAdapter.CreateChangesTable(schema.Tables[syncTable.TableName, syncTable.SchemaName], changesSet);
while (dataReader.Read())
{
// Create a row from dataReader
var row = CreateSyncRowFromReader(dataReader, changesSetTable);
// Add the row to the changes set
changesSetTable.Rows.Add(row);
// Log trace row
this.Orchestrator.logger.LogTrace(SyncEventsId.CreateSnapshot, row);
var fieldsSize = ContainerTable.GetRowSizeFromDataRow(row.ToArray());
var finalFieldSize = fieldsSize / 1024d;
if (finalFieldSize > batchSize)
{
throw new RowOverSizedException(finalFieldSize.ToString());
}
// Calculate the new memory size
rowsMemorySize += finalFieldSize;
// Next line if we don't reach the batch size yet.
if (rowsMemorySize <= batchSize)
{
continue;
}
// add changes to batchinfo
await batchInfo.AddChangesAsync(changesSet, batchIndex, false, this.Orchestrator).ConfigureAwait(false);
this.Orchestrator.logger.LogDebug(SyncEventsId.CreateBatch, changesSet);
// increment batch index
batchIndex++;
// we know the datas are serialized here, so we can flush the set
changesSet.Clear();
// Recreate an empty ContainerSet and a ContainerTable
changesSet = new SyncSet();
changesSetTable = SyncAdapter.CreateChangesTable(schema.Tables[syncTable.TableName, syncTable.SchemaName], changesSet);
// Init the row memory size
rowsMemorySize = 0L;
}
}
//selectIncrementalChangesCommand.Dispose();
}
if (changesSet != null && changesSet.HasTables)
{
await batchInfo.AddChangesAsync(changesSet, batchIndex, true, this.Orchestrator).ConfigureAwait(false);
this.Orchestrator.logger.LogDebug(SyncEventsId.CreateBatch, changesSet);
}
// Check the last index as the last batch
batchInfo.EnsureLastBatch();
batchInfo.Timestamp = remoteClientTimestamp;
// Serialize on disk.
var jsonConverter = new JsonConverter <BatchInfo>();
var summaryFileName = Path.Combine(directoryFullPath, "summary.json");
using (var f = new FileStream(summaryFileName, FileMode.CreateNew, FileAccess.ReadWrite))
{
this.Orchestrator.logger.LogDebug(SyncEventsId.CreateSnapshotSummary, batchInfo);
var bytes = await jsonConverter.SerializeAsync(batchInfo).ConfigureAwait(false);
f.Write(bytes, 0, bytes.Length);
}
return(context, batchInfo);
}
/// <summary>
/// Gets a batch of changes to synchronize when given batch size,
/// destination knowledge, and change data retriever parameters.
/// </summary>
/// <returns>A DbSyncContext object that will be used to retrieve the modified data.</returns>
public virtual async Task <(SyncContext, BatchInfo)> GetSnapshotAsync(
SyncContext context, SyncSet schema, string snapshotDirectory,
CancellationToken cancellationToken, IProgress <ProgressArgs> progress = null)
{
// TODO : Get a snapshot based on scope name
var sb = new StringBuilder();
var underscore = "";
if (context.Parameters != null)
{
foreach (var p in context.Parameters.OrderBy(p => p.Name))
{
var cleanValue = new string(p.Value.ToString().Where(char.IsLetterOrDigit).ToArray());
var cleanName = new string(p.Name.Where(char.IsLetterOrDigit).ToArray());
sb.Append($"{underscore}{cleanName}_{cleanValue}");
underscore = "_";
}
}
var directoryName = sb.ToString();
directoryName = string.IsNullOrEmpty(directoryName) ? "ALL" : directoryName;
this.Orchestrator.logger.LogDebug(SyncEventsId.GetSnapshot, new { DirectoryName = directoryName });
// cleansing scope name
var directoryScopeName = new string(context.ScopeName.Where(char.IsLetterOrDigit).ToArray());
// Get full path
var directoryFullPath = Path.Combine(snapshotDirectory, directoryScopeName, directoryName);
// if no snapshot present, just return null value.
if (!Directory.Exists(directoryFullPath))
{
this.Orchestrator.logger.LogDebug(SyncEventsId.DirectoryNotExists, new { DirectoryPath = directoryFullPath });
return(context, null);
}
// Serialize on disk.
var jsonConverter = new JsonConverter <BatchInfo>();
var summaryFileName = Path.Combine(directoryFullPath, "summary.json");
BatchInfo batchInfo = null;
// Create the schema changeset
var changesSet = new SyncSet();
// Create a Schema set without readonly columns, attached to memory changes
foreach (var table in schema.Tables)
{
SyncAdapter.CreateChangesTable(schema.Tables[table.TableName, table.SchemaName], changesSet);
}
using (var fs = new FileStream(summaryFileName, FileMode.Open, FileAccess.Read))
{
this.Orchestrator.logger.LogDebug(SyncEventsId.LoadSnapshotSummary, new { FileName = summaryFileName });
batchInfo = await jsonConverter.DeserializeAsync(fs).ConfigureAwait(false);
this.Orchestrator.logger.LogDebug(SyncEventsId.LoadSnapshotSummary, batchInfo);
}
batchInfo.SanitizedSchema = changesSet;
return(context, batchInfo);
}
/// <summary>
/// Handle a conflict
/// The int returned is the conflict count I need
/// </summary>
/// changeApplicationAction, conflictCount, resolvedRow, conflictApplyInt
internal async Task <(int conflictResolvedCount, SyncRow resolvedRow, int rowAppliedCount)> HandleConflictAsync(
Guid localScopeId, Guid senderScopeId,
SyncAdapter syncAdapter, SyncContext context, SyncConflict conflict,
ConflictResolutionPolicy policy, long lastTimestamp,
DbConnection connection, DbTransaction transaction)
{
SyncRow finalRow;
ApplyAction conflictApplyAction;
int rowAppliedCount = 0;
Guid? nullableSenderScopeId = senderScopeId;
(conflictApplyAction, finalRow, nullableSenderScopeId) = await this.GetConflictActionAsync(context, conflict, policy, senderScopeId, connection, transaction).ConfigureAwait(false);
// Conflict rollbacked by user
if (conflictApplyAction == ApplyAction.Rollback)
{
throw new RollbackException("Rollback action taken on conflict");
}
// Local provider wins, update metadata
if (conflictApplyAction == ApplyAction.Continue)
{
var isMergeAction = finalRow != null;
var row = isMergeAction ? finalRow : conflict.LocalRow;
// Conflict on a line that is not present on the datasource
if (row == null)
{
return(conflictResolvedCount : 1, finalRow, rowAppliedCount : 0);
}
// if we have a merge action, we apply the row on the server
if (isMergeAction)
{
// if merge, we update locally the row and let the update_scope_id set to null
var isUpdated = await syncAdapter.ApplyUpdateAsync(row, lastTimestamp, null, true, connection, transaction);
// We don't update metadatas so the row is updated (on server side)
// and is mark as updated locally.
// and will be returned back to sender, since it's a merge, and we need it on the client
if (!isUpdated)
{
throw new Exception("Can't update the merge row.");
}
}
finalRow = isMergeAction ? row : conflict.LocalRow;
// We don't do anything, since we let the original row. so we resolved one conflict but applied no rows
return(conflictResolvedCount : 1, finalRow, rowAppliedCount : 0);
}
// We gonna apply with force the line
if (conflictApplyAction == ApplyAction.RetryWithForceWrite)
{
// TODO : Should Raise an error ?
if (conflict.RemoteRow == null)
{
return(0, finalRow, 0);
}
bool operationComplete = false;
switch (conflict.Type)
{
// Remote source has row, Local don't have the row, so insert it
case ConflictType.RemoteExistsLocalExists:
operationComplete = await syncAdapter.ApplyUpdateAsync(conflict.RemoteRow, lastTimestamp, nullableSenderScopeId, true, connection, transaction);
rowAppliedCount = 1;
break;
case ConflictType.RemoteExistsLocalNotExists:
case ConflictType.RemoteExistsLocalIsDeleted:
case ConflictType.UniqueKeyConstraint:
operationComplete = await syncAdapter.ApplyUpdateAsync(conflict.RemoteRow, lastTimestamp, nullableSenderScopeId, true, connection, transaction);
rowAppliedCount = 1;
break;
// Conflict, but both have delete the row, so just update the metadata to the right winner
case ConflictType.RemoteIsDeletedLocalIsDeleted:
operationComplete = await syncAdapter.UpdateMetadatasAsync(conflict.RemoteRow, nullableSenderScopeId, true, connection, transaction);
rowAppliedCount = 0;
break;
// The row does not exists locally, and since it's coming from a deleted state, we can forget it
case ConflictType.RemoteIsDeletedLocalNotExists:
operationComplete = true;
rowAppliedCount = 0;
break;
// The remote has delete the row, and local has insert or update it
// So delete the local row
case ConflictType.RemoteIsDeletedLocalExists:
operationComplete = await syncAdapter.ApplyDeleteAsync(conflict.RemoteRow, lastTimestamp, nullableSenderScopeId, true, connection, transaction);
rowAppliedCount = 1;
break;
case ConflictType.ErrorsOccurred:
return(0, finalRow, 0);
}
finalRow = conflict.RemoteRow;
//After a force update, there is a problem, so raise exception
if (!operationComplete)
{
throw new UnknownException("Force update should always work.. contact the author :)");
//finalRow = null;
//return (0, finalRow, rowAppliedCount);
}
return(1, finalRow, rowAppliedCount);
}
return(0, finalRow, 0);
}
private async Task <(int rowsAppliedCount, int conflictsResolvedCount, int syncErrorsCount)> ResolveConflictsAsync(SyncContext context, Guid localScopeId, Guid senderScopeId, SyncAdapter syncAdapter, List <SyncConflict> conflicts, MessageApplyChanges message, DbConnection connection, DbTransaction transaction)
{
int rowsAppliedCount = 0;
int conflictsResolvedCount = 0;
int syncErrorsCount = 0;
// If conflicts occured
// Eventuall, conflicts are resolved on server side.
if (conflicts == null || conflicts.Count <= 0)
{
return(rowsAppliedCount, conflictsResolvedCount, syncErrorsCount);
}
foreach (var conflict in conflicts)
{
var fromScopeLocalTimeStamp = message.LastTimestamp;
this.Orchestrator.logger.LogDebug(SyncEventsId.ResolveConflicts, conflict);
this.Orchestrator.logger.LogDebug(SyncEventsId.ResolveConflicts, new
{
LocalScopeId = localScopeId,
SenderScopeId = senderScopeId,
FromScopeLocalTimeStamp = fromScopeLocalTimeStamp,
message.Policy
});;
var(conflictResolvedCount, resolvedRow, rowAppliedCount) =
await this.HandleConflictAsync(localScopeId, senderScopeId, syncAdapter, context, conflict,
message.Policy,
fromScopeLocalTimeStamp, connection, transaction).ConfigureAwait(false);
conflictsResolvedCount += conflictResolvedCount;
rowsAppliedCount += rowAppliedCount;
}
return(rowsAppliedCount, conflictsResolvedCount, syncErrorsCount);
}
/// <summary>
/// Get the correct Select changes command
/// Can be either
/// - SelectInitializedChanges : All changes for first sync
/// - SelectChanges : All changes filtered by timestamp
/// - SelectInitializedChangesWithFilters : All changes for first sync with filters
/// - SelectChangesWithFilters : All changes filtered by timestamp with filters
/// </summary>
private async Task <DbCommand> GetSelectChangesCommandAsync(SyncContext context, SyncAdapter syncAdapter, SyncTable syncTable, bool isNew, DbConnection connection, DbTransaction transaction)
{
DbCommand command;
DbCommandType dbCommandType;
SyncFilter tableFilter = null;
// Check if we have parameters specified
// Sqlite does not have any filter, since he can't be server side
if (this.CanBeServerProvider)
{
tableFilter = syncTable.GetFilter();
}
var hasFilters = tableFilter != null;
// Determing the correct DbCommandType
if (isNew && hasFilters)
{
dbCommandType = DbCommandType.SelectInitializedChangesWithFilters;
}
else if (isNew && !hasFilters)
{
dbCommandType = DbCommandType.SelectInitializedChanges;
}
else if (!isNew && hasFilters)
{
dbCommandType = DbCommandType.SelectChangesWithFilters;
}
else
{
dbCommandType = DbCommandType.SelectChanges;
}
// Get correct Select incremental changes command
command = await syncAdapter.PrepareCommandAsync(dbCommandType, connection, transaction, tableFilter);
return(command);
}
/// <summary>
/// Gets a batch of changes to synchronize when given batch size,
/// destination knowledge, and change data retriever parameters.
/// </summary>
/// <returns>A DbSyncContext object that will be used to retrieve the modified data.</returns>
public virtual async Task <(SyncContext, BatchInfo, DatabaseChangesSelected)> GetChangeBatchAsync(
SyncContext context, MessageGetChangesBatch message,
DbConnection connection, DbTransaction transaction,
CancellationToken cancellationToken, IProgress <ProgressArgs> progress)
{
// batch info containing changes
BatchInfo batchInfo;
this.Orchestrator.logger.LogDebug(SyncEventsId.GetChanges, message);
// Statistics about changes that are selected
DatabaseChangesSelected changesSelected;
if (context.SyncWay == SyncWay.Upload && context.SyncType == SyncType.Reinitialize)
{
(batchInfo, changesSelected) = await this.GetEmptyChangesAsync(message).ConfigureAwait(false);
return(context, batchInfo, changesSelected);
}
// create local directory
if (message.BatchSize > 0 && !string.IsNullOrEmpty(message.BatchDirectory) && !Directory.Exists(message.BatchDirectory))
{
this.Orchestrator.logger.LogDebug(SyncEventsId.CreateDirectory, new { DirectoryPath = message.BatchDirectory });
Directory.CreateDirectory(message.BatchDirectory);
}
// numbers of batch files generated
var batchIndex = 0;
// Check if we are in batch mode
var isBatch = message.BatchSize > 0;
// Create stats object to store changes count
var changes = new DatabaseChangesSelected();
// Create a batch info in memory (if !isBatch) or serialized on disk (if isBatch)
// batchinfo generate a schema clone with scope columns if needed
batchInfo = new BatchInfo(!isBatch, message.Schema, message.BatchDirectory);
// Clean SyncSet, we will add only tables we need in the batch info
var changesSet = new SyncSet();
foreach (var syncTable in message.Schema.Tables)
{
this.Orchestrator.logger.LogDebug(SyncEventsId.GetChanges, syncTable);
// if we are in upload stage, so check if table is not download only
if (context.SyncWay == SyncWay.Upload && syncTable.SyncDirection == SyncDirection.DownloadOnly)
{
continue;
}
// if we are in download stage, so check if table is not download only
if (context.SyncWay == SyncWay.Download && syncTable.SyncDirection == SyncDirection.UploadOnly)
{
continue;
}
var tableBuilder = this.GetTableBuilder(syncTable, message.Setup);
var syncAdapter = tableBuilder.CreateSyncAdapter();
// launch interceptor if any
await this.Orchestrator.InterceptAsync(new TableChangesSelectingArgs(context, syncTable, connection, transaction), cancellationToken).ConfigureAwait(false);
// Get Command
var selectIncrementalChangesCommand = await this.GetSelectChangesCommandAsync(context, syncAdapter, syncTable, message.IsNew, connection, transaction);
// Set parameters
this.SetSelectChangesCommonParameters(context, syncTable, message.ExcludingScopeId, message.IsNew, message.LastTimestamp, selectIncrementalChangesCommand);
// log
this.Orchestrator.logger.LogDebug(SyncEventsId.GetChanges, selectIncrementalChangesCommand);
// Statistics
var tableChangesSelected = new TableChangesSelected(syncTable.TableName, syncTable.SchemaName);
// Create a chnages table with scope columns
var changesSetTable = SyncAdapter.CreateChangesTable(message.Schema.Tables[syncTable.TableName, syncTable.SchemaName], changesSet);
// Get the reader
using (var dataReader = await selectIncrementalChangesCommand.ExecuteReaderAsync().ConfigureAwait(false))
{
// memory size total
double rowsMemorySize = 0L;
while (dataReader.Read())
{
// Create a row from dataReader
var row = CreateSyncRowFromReader(dataReader, changesSetTable);
// Add the row to the changes set
changesSetTable.Rows.Add(row);
// Log trace row
this.Orchestrator.logger.LogTrace(SyncEventsId.GetChanges, row);
// Set the correct state to be applied
if (row.RowState == DataRowState.Deleted)
{
tableChangesSelected.Deletes++;
}
else if (row.RowState == DataRowState.Modified)
{
tableChangesSelected.Upserts++;
}
// calculate row size if in batch mode
if (isBatch)
{
var fieldsSize = ContainerTable.GetRowSizeFromDataRow(row.ToArray());
var finalFieldSize = fieldsSize / 1024d;
if (finalFieldSize > message.BatchSize)
{
throw new RowOverSizedException(finalFieldSize.ToString());
}
// Calculate the new memory size
rowsMemorySize += finalFieldSize;
// Next line if we don't reach the batch size yet.
if (rowsMemorySize <= message.BatchSize)
{
continue;
}
// Check interceptor
var batchTableChangesSelectedArgs = new TableChangesSelectedArgs(context, changesSetTable, tableChangesSelected, connection, transaction);
await this.Orchestrator.InterceptAsync(batchTableChangesSelectedArgs, cancellationToken).ConfigureAwait(false);
// add changes to batchinfo
await batchInfo.AddChangesAsync(changesSet, batchIndex, false, this.Orchestrator).ConfigureAwait(false);
this.Orchestrator.logger.LogDebug(SyncEventsId.GetChanges, changesSet);
// increment batch index
batchIndex++;
// we know the datas are serialized here, so we can flush the set
changesSet.Clear();
// Recreate an empty ContainerSet and a ContainerTable
changesSet = new SyncSet();
changesSetTable = SyncAdapter.CreateChangesTable(message.Schema.Tables[syncTable.TableName, syncTable.SchemaName], changesSet);
// Init the row memory size
rowsMemorySize = 0L;
}
}
}
// be sure it's disposed
// selectIncrementalChangesCommand.Dispose();
// We don't report progress if no table changes is empty, to limit verbosity
if (tableChangesSelected.Deletes > 0 || tableChangesSelected.Upserts > 0)
{
changes.TableChangesSelected.Add(tableChangesSelected);
var tableChangesSelectedArgs = new TableChangesSelectedArgs(context, changesSetTable, tableChangesSelected, connection, transaction);
await this.Orchestrator.InterceptAsync(tableChangesSelectedArgs, cancellationToken).ConfigureAwait(false);
if (tableChangesSelectedArgs.TableChangesSelected.TotalChanges > 0)
{
this.Orchestrator.ReportProgress(context, progress, tableChangesSelectedArgs);
}
}
}
// We are in batch mode, and we are at the last batchpart info
// Even if we don't have rows inside, we return the changesSet, since it contains at least schema
if (changesSet != null && changesSet.HasTables && changesSet.HasRows)
{
await batchInfo.AddChangesAsync(changesSet, batchIndex, true, this.Orchestrator).ConfigureAwait(false);
this.Orchestrator.logger.LogDebug(SyncEventsId.GetChanges, changesSet);
}
// Check the last index as the last batch
batchInfo.EnsureLastBatch();
return(context, batchInfo, changes);
}
