/// <summary>
/// SyncAgent manage both server and client provider
/// the Configuration object represents the server configuration
/// Don't work on the proxy provider
/// </summary>
public SyncAgent(IProvider clientProvider, IProvider serverProvider, SyncConfiguration configuration)
: this("DefaultScope", clientProvider, serverProvider, configuration)
{
}
public SyncAgent(SyncConfiguration configuration)
{
this.Configuration = configuration;
}
/// <summary>
/// SyncAgent manage both server and client provider
/// the Configuration object represents the server configuration
/// Don't work on the proxy provider
/// </summary>
public SyncAgent(string scopeName, IProvider clientProvider, IProvider serverProvider, SyncConfiguration configuration)
: this(scopeName, clientProvider, serverProvider)
{
if (configuration.Count <= 0)
{
throw new SyncException("No tables specified", SyncStage.BeginSession, SyncExceptionType.Argument);
}
var remoteCoreProvider = this.RemoteProvider as CoreProvider;
if (remoteCoreProvider == null)
{
throw new ArgumentException("Since the remote provider is a web proxy, you have to configure the server side");
}
if (!remoteCoreProvider.CanBeServerProvider)
{
throw new NotSupportedException();
}
this.Configuration = configuration;
}
/// <summary>
/// Enumerate all internal changes, no batch mode
/// </summary>
internal async Task <(BatchInfo, ChangesSelected)> EnumerateChangesInBatchesInternal
(SyncContext context, ScopeInfo scopeInfo, int downloadBatchSizeInKB, DmSet configTables, string batchDirectory, ConflictResolutionPolicy policy, ICollection <FilterClause> filters)
{
DmTable dmTable = null;
// memory size total
double memorySizeFromDmRows = 0L;
var batchIndex = 0;
// this batch info won't be in memory, it will be be batched
var batchInfo = new BatchInfo(false, batchDirectory);
// directory where all files will be stored
batchInfo.GenerateNewDirectoryName();
// Create stats object to store changes count
var changes = new ChangesSelected();
using (var connection = this.CreateConnection())
{
try
{
// Open the connection
await connection.OpenAsync();
using (var transaction = connection.BeginTransaction())
{
// create the in memory changes set
var changesSet = new DmSet(configTables.DmSetName);
foreach (var tableDescription in configTables.Tables)
{
// if we are in upload stage, so check if table is not download only
if (context.SyncWay == SyncWay.Upload && tableDescription.SyncDirection == SyncDirection.DownloadOnly)
{
continue;
}
// if we are in download stage, so check if table is not download only
if (context.SyncWay == SyncWay.Download && tableDescription.SyncDirection == SyncDirection.UploadOnly)
{
continue;
}
var builder = this.GetDatabaseBuilder(tableDescription);
var syncAdapter = builder.CreateSyncAdapter(connection, transaction);
syncAdapter.ConflictApplyAction = SyncConfiguration.GetApplyAction(policy);
// raise before event
context.SyncStage = SyncStage.TableChangesSelecting;
var beforeArgs = new TableChangesSelectingEventArgs(this.ProviderTypeName, context.SyncStage, tableDescription.TableName, connection, transaction);
this.TryRaiseProgressEvent(beforeArgs, this.TableChangesSelecting);
// Get Command
DbCommand selectIncrementalChangesCommand;
DbCommandType dbCommandType;
if (this.CanBeServerProvider && context.Parameters != null && context.Parameters.Count > 0 && filters != null && filters.Count > 0)
{
var filtersName = filters
.Where(f => f.TableName.Equals(tableDescription.TableName, StringComparison.InvariantCultureIgnoreCase))
.Select(f => f.ColumnName);
if (filtersName != null && filtersName.Count() > 0)
{
dbCommandType = DbCommandType.SelectChangesWitFilters;
selectIncrementalChangesCommand = syncAdapter.GetCommand(dbCommandType, filtersName);
}
else
{
dbCommandType = DbCommandType.SelectChanges;
selectIncrementalChangesCommand = syncAdapter.GetCommand(dbCommandType);
}
}
else
{
dbCommandType = DbCommandType.SelectChanges;
selectIncrementalChangesCommand = syncAdapter.GetCommand(dbCommandType);
}
// Deriving Parameters
syncAdapter.SetCommandParameters(DbCommandType.SelectChanges, selectIncrementalChangesCommand);
if (selectIncrementalChangesCommand == null)
{
var exc = "Missing command 'SelectIncrementalChangesCommand' ";
throw new Exception(exc);
}
dmTable = this.BuildChangesTable(tableDescription.TableName, configTables);
try
{
// Set commons parameters
SetSelectChangesCommonParameters(context, scopeInfo, selectIncrementalChangesCommand);
// Set filter parameters if any
// Only on server side
if (this.CanBeServerProvider && context.Parameters != null && context.Parameters.Count > 0 && filters != null && filters.Count > 0)
{
var filterTable = filters.Where(f => f.TableName.Equals(tableDescription.TableName, StringComparison.InvariantCultureIgnoreCase)).ToList();
if (filterTable != null && filterTable.Count > 0)
{
foreach (var filter in filterTable)
{
var parameter = context.Parameters.FirstOrDefault(p => p.ColumnName.Equals(filter.ColumnName, StringComparison.InvariantCultureIgnoreCase) && p.TableName.Equals(filter.TableName, StringComparison.InvariantCultureIgnoreCase));
if (parameter != null)
{
DbManager.SetParameterValue(selectIncrementalChangesCommand, parameter.ColumnName, parameter.Value);
}
}
}
}
this.AddTrackingColumns <int>(dmTable, "sync_row_is_tombstone");
// Statistics
var tableChangesSelected = new TableChangesSelected
{
TableName = tableDescription.TableName
};
changes.TableChangesSelected.Add(tableChangesSelected);
// Get the reader
using (var dataReader = selectIncrementalChangesCommand.ExecuteReader())
{
while (dataReader.Read())
{
var dmRow = this.CreateRowFromReader(dataReader, dmTable);
var state = DmRowState.Unchanged;
state = this.GetStateFromDmRow(dmRow, scopeInfo);
// If the row is not deleted inserted or modified, go next
if (state != DmRowState.Deleted && state != DmRowState.Modified && state != DmRowState.Added)
{
continue;
}
var fieldsSize = DmTableSurrogate.GetRowSizeFromDataRow(dmRow);
var dmRowSize = fieldsSize / 1024d;
if (dmRowSize > downloadBatchSizeInKB)
{
var exc = $"Row is too big ({dmRowSize} kb.) for the current Configuration.DownloadBatchSizeInKB ({downloadBatchSizeInKB} kb.) Aborting Sync...";
throw new Exception(exc);
}
// Calculate the new memory size
memorySizeFromDmRows = memorySizeFromDmRows + dmRowSize;
// add row
dmTable.Rows.Add(dmRow);
tableChangesSelected.TotalChanges++;
// acceptchanges before modifying
dmRow.AcceptChanges();
// Set the correct state to be applied
if (state == DmRowState.Deleted)
{
dmRow.Delete();
tableChangesSelected.Deletes++;
}
else if (state == DmRowState.Added)
{
dmRow.SetAdded();
tableChangesSelected.Inserts++;
}
else if (state == DmRowState.Modified)
{
dmRow.SetModified();
tableChangesSelected.Updates++;
}
// We exceed the memorySize, so we can add it to a batch
if (memorySizeFromDmRows > downloadBatchSizeInKB)
{
// Since we dont need this column anymore, remove it
this.RemoveTrackingColumns(dmTable, "sync_row_is_tombstone");
changesSet.Tables.Add(dmTable);
// generate the batch part info
batchInfo.GenerateBatchInfo(batchIndex, changesSet);
// increment batch index
batchIndex++;
changesSet.Clear();
// Recreate an empty DmSet, then a dmTable clone
changesSet = new DmSet(configTables.DmSetName);
dmTable = dmTable.Clone();
this.AddTrackingColumns <int>(dmTable, "sync_row_is_tombstone");
// Init the row memory size
memorySizeFromDmRows = 0L;
// add stats for a SyncProgress event
context.SyncStage = SyncStage.TableChangesSelected;
var args2 = new TableChangesSelectedEventArgs
(this.ProviderTypeName, SyncStage.TableChangesSelected, tableChangesSelected, connection, transaction);
this.TryRaiseProgressEvent(args2, this.TableChangesSelected);
}
}
// Since we dont need this column anymore, remove it
this.RemoveTrackingColumns(dmTable, "sync_row_is_tombstone");
context.SyncStage = SyncStage.TableChangesSelected;
changesSet.Tables.Add(dmTable);
// Init the row memory size
memorySizeFromDmRows = 0L;
// Event progress
context.SyncStage = SyncStage.TableChangesSelected;
var args = new TableChangesSelectedEventArgs(this.ProviderTypeName, SyncStage.TableChangesSelected, tableChangesSelected, connection, transaction);
this.TryRaiseProgressEvent(args, this.TableChangesSelected);
}
}
catch (Exception)
{
throw;
}
finally
{
}
}
// We are in batch mode, and we are at the last batchpart info
if (changesSet != null && changesSet.HasTables && changesSet.HasChanges())
{
var batchPartInfo = batchInfo.GenerateBatchInfo(batchIndex, changesSet);
if (batchPartInfo != null)
{
batchPartInfo.IsLastBatch = true;
}
}
transaction.Commit();
}
}
catch (Exception)
{
throw;
}
finally
{
if (connection != null && connection.State == ConnectionState.Open)
{
connection.Close();
}
}
}
return(batchInfo, changes);
}
/// <summary>
/// Enumerate all internal changes, no batch mode
/// </summary>
internal async Task <(BatchInfo, ChangesSelected)> EnumerateChangesInternal(
SyncContext context, ScopeInfo scopeInfo, DmSet configTables, string batchDirectory, ConflictResolutionPolicy policy, ICollection <FilterClause> filters)
{
// create the in memory changes set
var changesSet = new DmSet(SyncConfiguration.DMSET_NAME);
// Create the batch info, in memory
// No need to geneate a directory name, since we are in memory
var batchInfo = new BatchInfo(true, batchDirectory);
using (var connection = this.CreateConnection())
{
// Open the connection
await connection.OpenAsync();
using (var transaction = connection.BeginTransaction())
{
try
{
// changes that will be returned as selected changes
var changes = new ChangesSelected();
foreach (var tableDescription in configTables.Tables)
{
// if we are in upload stage, so check if table is not download only
if (context.SyncWay == SyncWay.Upload && tableDescription.SyncDirection == SyncDirection.DownloadOnly)
{
continue;
}
// if we are in download stage, so check if table is not download only
if (context.SyncWay == SyncWay.Download && tableDescription.SyncDirection == SyncDirection.UploadOnly)
{
continue;
}
var builder = this.GetDatabaseBuilder(tableDescription);
var syncAdapter = builder.CreateSyncAdapter(connection, transaction);
syncAdapter.ConflictApplyAction = SyncConfiguration.GetApplyAction(policy);
// raise before event
context.SyncStage = SyncStage.TableChangesSelecting;
var beforeArgs = new TableChangesSelectingEventArgs(this.ProviderTypeName, context.SyncStage, tableDescription.TableName, connection, transaction);
this.TryRaiseProgressEvent(beforeArgs, this.TableChangesSelecting);
// selected changes for the current table
var tableSelectedChanges = new TableChangesSelected
{
TableName = tableDescription.TableName
};
// Get Command
DbCommand selectIncrementalChangesCommand;
DbCommandType dbCommandType;
if (this.CanBeServerProvider && context.Parameters != null && context.Parameters.Count > 0 && filters != null && filters.Count > 0)
{
var filtersName = filters
.Where(f => f.TableName.Equals(tableDescription.TableName, StringComparison.InvariantCultureIgnoreCase))
.Select(f => f.ColumnName);
if (filtersName != null && filtersName.Count() > 0)
{
dbCommandType = DbCommandType.SelectChangesWitFilters;
selectIncrementalChangesCommand = syncAdapter.GetCommand(dbCommandType, filtersName);
}
else
{
dbCommandType = DbCommandType.SelectChanges;
selectIncrementalChangesCommand = syncAdapter.GetCommand(dbCommandType);
}
}
else
{
dbCommandType = DbCommandType.SelectChanges;
selectIncrementalChangesCommand = syncAdapter.GetCommand(dbCommandType);
}
if (selectIncrementalChangesCommand == null)
{
var exc = "Missing command 'SelectIncrementalChangesCommand' ";
throw new Exception(exc);
}
// Deriving Parameters
syncAdapter.SetCommandParameters(dbCommandType, selectIncrementalChangesCommand);
// Get a clone of the table with tracking columns
var dmTableChanges = this.BuildChangesTable(tableDescription.TableName, configTables);
SetSelectChangesCommonParameters(context, scopeInfo, selectIncrementalChangesCommand);
// Set filter parameters if any
if (this.CanBeServerProvider && context.Parameters != null && context.Parameters.Count > 0 && filters != null && filters.Count > 0)
{
var tableFilters = filters.Where(f => f.TableName.Equals(tableDescription.TableName, StringComparison.InvariantCultureIgnoreCase)).ToList();
if (tableFilters != null && tableFilters.Count > 0)
{
foreach (var filter in tableFilters)
{
var parameter = context.Parameters.FirstOrDefault(p => p.ColumnName.Equals(filter.ColumnName, StringComparison.InvariantCultureIgnoreCase) && p.TableName.Equals(filter.TableName, StringComparison.InvariantCultureIgnoreCase));
if (parameter != null)
{
DbManager.SetParameterValue(selectIncrementalChangesCommand, parameter.ColumnName, parameter.Value);
}
}
}
}
this.AddTrackingColumns <int>(dmTableChanges, "sync_row_is_tombstone");
// Get the reader
using (var dataReader = selectIncrementalChangesCommand.ExecuteReader())
{
while (dataReader.Read())
{
var dataRow = this.CreateRowFromReader(dataReader, dmTableChanges);
//DmRow dataRow = dmTableChanges.NewRow();
// assuming the row is not inserted / modified
var state = DmRowState.Unchanged;
// get if the current row is inserted, modified, deleted
state = this.GetStateFromDmRow(dataRow, scopeInfo);
if (state != DmRowState.Deleted && state != DmRowState.Modified && state != DmRowState.Added)
{
continue;
}
// add row
dmTableChanges.Rows.Add(dataRow);
// acceptchanges before modifying
dataRow.AcceptChanges();
tableSelectedChanges.TotalChanges++;
// Set the correct state to be applied
if (state == DmRowState.Deleted)
{
dataRow.Delete();
tableSelectedChanges.Deletes++;
}
else if (state == DmRowState.Added)
{
dataRow.SetAdded();
tableSelectedChanges.Inserts++;
}
else if (state == DmRowState.Modified)
{
dataRow.SetModified();
tableSelectedChanges.Updates++;
}
}
// Since we dont need this column anymore, remove it
this.RemoveTrackingColumns(dmTableChanges, "sync_row_is_tombstone");
// add it to the DmSet
changesSet.Tables.Add(dmTableChanges);
}
// add the stats to global stats
changes.TableChangesSelected.Add(tableSelectedChanges);
// Raise event for this table
context.SyncStage = SyncStage.TableChangesSelected;
var args = new TableChangesSelectedEventArgs(this.ProviderTypeName, SyncStage.TableChangesSelected, tableSelectedChanges, connection, transaction);
this.TryRaiseProgressEvent(args, this.TableChangesSelected);
}
transaction.Commit();
// generate the batchpartinfo
batchInfo.GenerateBatchInfo(0, changesSet);
// Create a new in-memory batch info with an the changes DmSet
return(batchInfo, changes);
}
catch (Exception)
{
throw;
}
finally
{
if (connection != null && connection.State == ConnectionState.Open)
{
connection.Close();
}
}
}
}
}
internal static void SerializeInDmSet(DmSet set, SyncConfiguration configuration)
{
if (set == null)
{
return;
}
DmTable dmTableConfiguration = null;
DmTable dmTableFilterParameters = null;
if (!set.Tables.Contains("DotmimSync__ServiceConfiguration"))
{
dmTableConfiguration = new DmTable("DotmimSync__ServiceConfiguration");
set.Tables.Add(dmTableConfiguration);
}
dmTableConfiguration = set.Tables["DotmimSync__ServiceConfiguration"];
dmTableConfiguration.Clear();
dmTableConfiguration.Columns.Clear();
dmTableConfiguration.Columns.Add <String>("BatchDirectory");
dmTableConfiguration.Columns.Add <Int32>("ConflictResolutionPolicy");
dmTableConfiguration.Columns.Add <Int32>("DownloadBatchSizeInKB");
dmTableConfiguration.Columns.Add <Boolean>("EnableDiagnosticPage");
dmTableConfiguration.Columns.Add <Boolean>("UseBulkOperations");
dmTableConfiguration.Columns.Add <Boolean>("UseVerboseErrors");
//dmTableConfiguration.Columns.Add<Boolean>("OverwriteConfiguration");
dmTableConfiguration.Columns.Add <Int32>("SerializationConverter");
var dmRowConfiguration = dmTableConfiguration.NewRow();
dmRowConfiguration["BatchDirectory"] = configuration.BatchDirectory;
dmRowConfiguration["ConflictResolutionPolicy"] = configuration.ConflictResolutionPolicy;
dmRowConfiguration["DownloadBatchSizeInKB"] = configuration.DownloadBatchSizeInKB;
dmRowConfiguration["UseBulkOperations"] = configuration.UseBulkOperations;
dmRowConfiguration["UseVerboseErrors"] = configuration.UseVerboseErrors;
//dmRowConfiguration["OverwriteConfiguration"] = configuration.OverwriteConfiguration;
dmRowConfiguration["SerializationConverter"] = configuration.SerializationFormat;
dmTableConfiguration.Rows.Add(dmRowConfiguration);
if (configuration.ScopeSet != null && configuration.ScopeSet.Tables.Count > 0)
{
foreach (var dmTable in configuration.ScopeSet.Tables)
{
var dmTableConf = dmTable.Clone();
set.Tables.Add(dmTableConf);
}
foreach (var dmRelation in configuration.ScopeSet.Relations)
{
var dmRelationConf = dmRelation.Clone(set);
set.Relations.Add(dmRelationConf);
}
}
if (configuration.Filters.Count > 0)
{
if (!set.Tables.Contains("DotmimSync__Filters"))
{
dmTableFilterParameters = new DmTable("DotmimSync__Filters");
set.Tables.Add(dmTableFilterParameters);
}
dmTableFilterParameters = set.Tables["DotmimSync__Filters"];
dmTableFilterParameters.Columns.Add <String>("TableName");
dmTableFilterParameters.Columns.Add <String>("ColumnName");
foreach (var p in configuration.Filters)
{
var dmRowFilter = dmTableFilterParameters.NewRow();
dmRowFilter["TableName"] = p.TableName;
dmRowFilter["ColumnName"] = p.ColumnName;
dmTableFilterParameters.Rows.Add(dmRowFilter);
}
}
}
internal static SyncConfiguration DeserializeFromDmSet(DmSet set)
{
if (set == null)
{
return(null);
}
if (!set.Tables.Contains("DotmimSync__ServiceConfiguration"))
{
return(null);
}
SyncConfiguration configuration = new SyncConfiguration();
var dmRowConfiguration = set.Tables["DotmimSync__ServiceConfiguration"].Rows[0];
configuration.BatchDirectory = dmRowConfiguration["BatchDirectory"] as String;;
configuration.ConflictResolutionPolicy = (ConflictResolutionPolicy)dmRowConfiguration["ConflictResolutionPolicy"];
configuration.DownloadBatchSizeInKB = (int)dmRowConfiguration["DownloadBatchSizeInKB"];
configuration.UseBulkOperations = (bool)dmRowConfiguration["UseBulkOperations"];
configuration.UseVerboseErrors = (bool)dmRowConfiguration["UseVerboseErrors"];
configuration.OverwriteConfiguration = (bool)dmRowConfiguration["OverwriteConfiguration"];
configuration.SerializationFormat = (SerializationFormat)dmRowConfiguration["SerializationConverter"];
if (set.Tables.Contains("DotmimSync__Filters"))
{
var dmTableFilterParameters = set.Tables["DotmimSync__Filters"];
foreach (var dmRowFilter in dmTableFilterParameters.Rows)
{
FilterClause filterClause = new FilterClause();
var tableName = dmRowFilter["TableName"] as String;
var columnName = dmRowFilter["ColumnName"] as String;
configuration.Filters.Add(tableName, columnName);
}
}
//if (set.Tables.Contains("DotmimSync__Table"))
//{
// var dmTableTables = set.Tables["DotmimSync__Table"];
// configuration.Tables = new string[dmTableTables.Rows.Count];
// for (int i = 0; i < dmTableTables.Rows.Count; i++)
// {
// var dmRowTable = dmTableTables.Rows[i];
// var tableName = dmRowTable["Name"] as String;
// configuration.Tables[i] = tableName;
// }
//}
var configTables = set.Tables.Where(tbl => !tbl.TableName.StartsWith("DotmimSync__"));
if (configTables != null)
{
foreach (var configTable in configTables)
{
configuration.ScopeSet.Tables.Add(configTable.Clone());
}
}
if (set.Relations != null && set.Relations.Count > 0)
{
foreach (var r in set.Relations)
{
var relation = r.Clone(configuration.ScopeSet);
configuration.ScopeSet.Relations.Add(relation);
}
}
return(configuration);
}
