/// <summary> /// SyncAgent manage both server and client provider /// the Configuration object represents the server configuration /// Don't work on the proxy provider /// </summary> public SyncAgent(IProvider clientProvider, IProvider serverProvider, SyncConfiguration configuration) : this("DefaultScope", clientProvider, serverProvider, configuration) { }
public SyncAgent(SyncConfiguration configuration) { this.Configuration = configuration; }
/// <summary> /// SyncAgent manage both server and client provider /// the Configuration object represents the server configuration /// Don't work on the proxy provider /// </summary> public SyncAgent(string scopeName, IProvider clientProvider, IProvider serverProvider, SyncConfiguration configuration) : this(scopeName, clientProvider, serverProvider) { if (configuration.Count <= 0) { throw new SyncException("No tables specified", SyncStage.BeginSession, SyncExceptionType.Argument); } var remoteCoreProvider = this.RemoteProvider as CoreProvider; if (remoteCoreProvider == null) { throw new ArgumentException("Since the remote provider is a web proxy, you have to configure the server side"); } if (!remoteCoreProvider.CanBeServerProvider) { throw new NotSupportedException(); } this.Configuration = configuration; }
/// <summary> /// Enumerate all internal changes, no batch mode /// </summary> internal async Task <(BatchInfo, ChangesSelected)> EnumerateChangesInBatchesInternal (SyncContext context, ScopeInfo scopeInfo, int downloadBatchSizeInKB, DmSet configTables, string batchDirectory, ConflictResolutionPolicy policy, ICollection <FilterClause> filters) { DmTable dmTable = null; // memory size total double memorySizeFromDmRows = 0L; var batchIndex = 0; // this batch info won't be in memory, it will be be batched var batchInfo = new BatchInfo(false, batchDirectory); // directory where all files will be stored batchInfo.GenerateNewDirectoryName(); // Create stats object to store changes count var changes = new ChangesSelected(); using (var connection = this.CreateConnection()) { try { // Open the connection await connection.OpenAsync(); using (var transaction = connection.BeginTransaction()) { // create the in memory changes set var changesSet = new DmSet(configTables.DmSetName); foreach (var tableDescription in configTables.Tables) { // if we are in upload stage, so check if table is not download only if (context.SyncWay == SyncWay.Upload && tableDescription.SyncDirection == SyncDirection.DownloadOnly) { continue; } // if we are in download stage, so check if table is not download only if (context.SyncWay == SyncWay.Download && tableDescription.SyncDirection == SyncDirection.UploadOnly) { continue; } var builder = this.GetDatabaseBuilder(tableDescription); var syncAdapter = builder.CreateSyncAdapter(connection, transaction); syncAdapter.ConflictApplyAction = SyncConfiguration.GetApplyAction(policy); // raise before event context.SyncStage = SyncStage.TableChangesSelecting; var beforeArgs = new TableChangesSelectingEventArgs(this.ProviderTypeName, context.SyncStage, tableDescription.TableName, connection, transaction); this.TryRaiseProgressEvent(beforeArgs, this.TableChangesSelecting); // Get Command DbCommand selectIncrementalChangesCommand; DbCommandType dbCommandType; if (this.CanBeServerProvider && context.Parameters != null && context.Parameters.Count > 0 && filters != null && filters.Count > 0) { var filtersName = filters .Where(f => f.TableName.Equals(tableDescription.TableName, StringComparison.InvariantCultureIgnoreCase)) .Select(f => f.ColumnName); if (filtersName != null && filtersName.Count() > 0) { dbCommandType = DbCommandType.SelectChangesWitFilters; selectIncrementalChangesCommand = syncAdapter.GetCommand(dbCommandType, filtersName); } else { dbCommandType = DbCommandType.SelectChanges; selectIncrementalChangesCommand = syncAdapter.GetCommand(dbCommandType); } } else { dbCommandType = DbCommandType.SelectChanges; selectIncrementalChangesCommand = syncAdapter.GetCommand(dbCommandType); } // Deriving Parameters syncAdapter.SetCommandParameters(DbCommandType.SelectChanges, selectIncrementalChangesCommand); if (selectIncrementalChangesCommand == null) { var exc = "Missing command 'SelectIncrementalChangesCommand' "; throw new Exception(exc); } dmTable = this.BuildChangesTable(tableDescription.TableName, configTables); try { // Set commons parameters SetSelectChangesCommonParameters(context, scopeInfo, selectIncrementalChangesCommand); // Set filter parameters if any // Only on server side if (this.CanBeServerProvider && context.Parameters != null && context.Parameters.Count > 0 && filters != null && filters.Count > 0) { var filterTable = filters.Where(f => f.TableName.Equals(tableDescription.TableName, StringComparison.InvariantCultureIgnoreCase)).ToList(); if (filterTable != null && filterTable.Count > 0) { foreach (var filter in filterTable) { var parameter = context.Parameters.FirstOrDefault(p => p.ColumnName.Equals(filter.ColumnName, StringComparison.InvariantCultureIgnoreCase) && p.TableName.Equals(filter.TableName, StringComparison.InvariantCultureIgnoreCase)); if (parameter != null) { DbManager.SetParameterValue(selectIncrementalChangesCommand, parameter.ColumnName, parameter.Value); } } } } this.AddTrackingColumns <int>(dmTable, "sync_row_is_tombstone"); // Statistics var tableChangesSelected = new TableChangesSelected { TableName = tableDescription.TableName }; changes.TableChangesSelected.Add(tableChangesSelected); // Get the reader using (var dataReader = selectIncrementalChangesCommand.ExecuteReader()) { while (dataReader.Read()) { var dmRow = this.CreateRowFromReader(dataReader, dmTable); var state = DmRowState.Unchanged; state = this.GetStateFromDmRow(dmRow, scopeInfo); // If the row is not deleted inserted or modified, go next if (state != DmRowState.Deleted && state != DmRowState.Modified && state != DmRowState.Added) { continue; } var fieldsSize = DmTableSurrogate.GetRowSizeFromDataRow(dmRow); var dmRowSize = fieldsSize / 1024d; if (dmRowSize > downloadBatchSizeInKB) { var exc = $"Row is too big ({dmRowSize} kb.) for the current Configuration.DownloadBatchSizeInKB ({downloadBatchSizeInKB} kb.) Aborting Sync..."; throw new Exception(exc); } // Calculate the new memory size memorySizeFromDmRows = memorySizeFromDmRows + dmRowSize; // add row dmTable.Rows.Add(dmRow); tableChangesSelected.TotalChanges++; // acceptchanges before modifying dmRow.AcceptChanges(); // Set the correct state to be applied if (state == DmRowState.Deleted) { dmRow.Delete(); tableChangesSelected.Deletes++; } else if (state == DmRowState.Added) { dmRow.SetAdded(); tableChangesSelected.Inserts++; } else if (state == DmRowState.Modified) { dmRow.SetModified(); tableChangesSelected.Updates++; } // We exceed the memorySize, so we can add it to a batch if (memorySizeFromDmRows > downloadBatchSizeInKB) { // Since we dont need this column anymore, remove it this.RemoveTrackingColumns(dmTable, "sync_row_is_tombstone"); changesSet.Tables.Add(dmTable); // generate the batch part info batchInfo.GenerateBatchInfo(batchIndex, changesSet); // increment batch index batchIndex++; changesSet.Clear(); // Recreate an empty DmSet, then a dmTable clone changesSet = new DmSet(configTables.DmSetName); dmTable = dmTable.Clone(); this.AddTrackingColumns <int>(dmTable, "sync_row_is_tombstone"); // Init the row memory size memorySizeFromDmRows = 0L; // add stats for a SyncProgress event context.SyncStage = SyncStage.TableChangesSelected; var args2 = new TableChangesSelectedEventArgs (this.ProviderTypeName, SyncStage.TableChangesSelected, tableChangesSelected, connection, transaction); this.TryRaiseProgressEvent(args2, this.TableChangesSelected); } } // Since we dont need this column anymore, remove it this.RemoveTrackingColumns(dmTable, "sync_row_is_tombstone"); context.SyncStage = SyncStage.TableChangesSelected; changesSet.Tables.Add(dmTable); // Init the row memory size memorySizeFromDmRows = 0L; // Event progress context.SyncStage = SyncStage.TableChangesSelected; var args = new TableChangesSelectedEventArgs(this.ProviderTypeName, SyncStage.TableChangesSelected, tableChangesSelected, connection, transaction); this.TryRaiseProgressEvent(args, this.TableChangesSelected); } } catch (Exception) { throw; } finally { } } // We are in batch mode, and we are at the last batchpart info if (changesSet != null && changesSet.HasTables && changesSet.HasChanges()) { var batchPartInfo = batchInfo.GenerateBatchInfo(batchIndex, changesSet); if (batchPartInfo != null) { batchPartInfo.IsLastBatch = true; } } transaction.Commit(); } } catch (Exception) { throw; } finally { if (connection != null && connection.State == ConnectionState.Open) { connection.Close(); } } } return(batchInfo, changes); }
/// <summary> /// Enumerate all internal changes, no batch mode /// </summary> internal async Task <(BatchInfo, ChangesSelected)> EnumerateChangesInternal( SyncContext context, ScopeInfo scopeInfo, DmSet configTables, string batchDirectory, ConflictResolutionPolicy policy, ICollection <FilterClause> filters) { // create the in memory changes set var changesSet = new DmSet(SyncConfiguration.DMSET_NAME); // Create the batch info, in memory // No need to geneate a directory name, since we are in memory var batchInfo = new BatchInfo(true, batchDirectory); using (var connection = this.CreateConnection()) { // Open the connection await connection.OpenAsync(); using (var transaction = connection.BeginTransaction()) { try { // changes that will be returned as selected changes var changes = new ChangesSelected(); foreach (var tableDescription in configTables.Tables) { // if we are in upload stage, so check if table is not download only if (context.SyncWay == SyncWay.Upload && tableDescription.SyncDirection == SyncDirection.DownloadOnly) { continue; } // if we are in download stage, so check if table is not download only if (context.SyncWay == SyncWay.Download && tableDescription.SyncDirection == SyncDirection.UploadOnly) { continue; } var builder = this.GetDatabaseBuilder(tableDescription); var syncAdapter = builder.CreateSyncAdapter(connection, transaction); syncAdapter.ConflictApplyAction = SyncConfiguration.GetApplyAction(policy); // raise before event context.SyncStage = SyncStage.TableChangesSelecting; var beforeArgs = new TableChangesSelectingEventArgs(this.ProviderTypeName, context.SyncStage, tableDescription.TableName, connection, transaction); this.TryRaiseProgressEvent(beforeArgs, this.TableChangesSelecting); // selected changes for the current table var tableSelectedChanges = new TableChangesSelected { TableName = tableDescription.TableName }; // Get Command DbCommand selectIncrementalChangesCommand; DbCommandType dbCommandType; if (this.CanBeServerProvider && context.Parameters != null && context.Parameters.Count > 0 && filters != null && filters.Count > 0) { var filtersName = filters .Where(f => f.TableName.Equals(tableDescription.TableName, StringComparison.InvariantCultureIgnoreCase)) .Select(f => f.ColumnName); if (filtersName != null && filtersName.Count() > 0) { dbCommandType = DbCommandType.SelectChangesWitFilters; selectIncrementalChangesCommand = syncAdapter.GetCommand(dbCommandType, filtersName); } else { dbCommandType = DbCommandType.SelectChanges; selectIncrementalChangesCommand = syncAdapter.GetCommand(dbCommandType); } } else { dbCommandType = DbCommandType.SelectChanges; selectIncrementalChangesCommand = syncAdapter.GetCommand(dbCommandType); } if (selectIncrementalChangesCommand == null) { var exc = "Missing command 'SelectIncrementalChangesCommand' "; throw new Exception(exc); } // Deriving Parameters syncAdapter.SetCommandParameters(dbCommandType, selectIncrementalChangesCommand); // Get a clone of the table with tracking columns var dmTableChanges = this.BuildChangesTable(tableDescription.TableName, configTables); SetSelectChangesCommonParameters(context, scopeInfo, selectIncrementalChangesCommand); // Set filter parameters if any if (this.CanBeServerProvider && context.Parameters != null && context.Parameters.Count > 0 && filters != null && filters.Count > 0) { var tableFilters = filters.Where(f => f.TableName.Equals(tableDescription.TableName, StringComparison.InvariantCultureIgnoreCase)).ToList(); if (tableFilters != null && tableFilters.Count > 0) { foreach (var filter in tableFilters) { var parameter = context.Parameters.FirstOrDefault(p => p.ColumnName.Equals(filter.ColumnName, StringComparison.InvariantCultureIgnoreCase) && p.TableName.Equals(filter.TableName, StringComparison.InvariantCultureIgnoreCase)); if (parameter != null) { DbManager.SetParameterValue(selectIncrementalChangesCommand, parameter.ColumnName, parameter.Value); } } } } this.AddTrackingColumns <int>(dmTableChanges, "sync_row_is_tombstone"); // Get the reader using (var dataReader = selectIncrementalChangesCommand.ExecuteReader()) { while (dataReader.Read()) { var dataRow = this.CreateRowFromReader(dataReader, dmTableChanges); //DmRow dataRow = dmTableChanges.NewRow(); // assuming the row is not inserted / modified var state = DmRowState.Unchanged; // get if the current row is inserted, modified, deleted state = this.GetStateFromDmRow(dataRow, scopeInfo); if (state != DmRowState.Deleted && state != DmRowState.Modified && state != DmRowState.Added) { continue; } // add row dmTableChanges.Rows.Add(dataRow); // acceptchanges before modifying dataRow.AcceptChanges(); tableSelectedChanges.TotalChanges++; // Set the correct state to be applied if (state == DmRowState.Deleted) { dataRow.Delete(); tableSelectedChanges.Deletes++; } else if (state == DmRowState.Added) { dataRow.SetAdded(); tableSelectedChanges.Inserts++; } else if (state == DmRowState.Modified) { dataRow.SetModified(); tableSelectedChanges.Updates++; } } // Since we dont need this column anymore, remove it this.RemoveTrackingColumns(dmTableChanges, "sync_row_is_tombstone"); // add it to the DmSet changesSet.Tables.Add(dmTableChanges); } // add the stats to global stats changes.TableChangesSelected.Add(tableSelectedChanges); // Raise event for this table context.SyncStage = SyncStage.TableChangesSelected; var args = new TableChangesSelectedEventArgs(this.ProviderTypeName, SyncStage.TableChangesSelected, tableSelectedChanges, connection, transaction); this.TryRaiseProgressEvent(args, this.TableChangesSelected); } transaction.Commit(); // generate the batchpartinfo batchInfo.GenerateBatchInfo(0, changesSet); // Create a new in-memory batch info with an the changes DmSet return(batchInfo, changes); } catch (Exception) { throw; } finally { if (connection != null && connection.State == ConnectionState.Open) { connection.Close(); } } } } }
internal static void SerializeInDmSet(DmSet set, SyncConfiguration configuration) { if (set == null) { return; } DmTable dmTableConfiguration = null; DmTable dmTableFilterParameters = null; if (!set.Tables.Contains("DotmimSync__ServiceConfiguration")) { dmTableConfiguration = new DmTable("DotmimSync__ServiceConfiguration"); set.Tables.Add(dmTableConfiguration); } dmTableConfiguration = set.Tables["DotmimSync__ServiceConfiguration"]; dmTableConfiguration.Clear(); dmTableConfiguration.Columns.Clear(); dmTableConfiguration.Columns.Add <String>("BatchDirectory"); dmTableConfiguration.Columns.Add <Int32>("ConflictResolutionPolicy"); dmTableConfiguration.Columns.Add <Int32>("DownloadBatchSizeInKB"); dmTableConfiguration.Columns.Add <Boolean>("EnableDiagnosticPage"); dmTableConfiguration.Columns.Add <Boolean>("UseBulkOperations"); dmTableConfiguration.Columns.Add <Boolean>("UseVerboseErrors"); //dmTableConfiguration.Columns.Add<Boolean>("OverwriteConfiguration"); dmTableConfiguration.Columns.Add <Int32>("SerializationConverter"); var dmRowConfiguration = dmTableConfiguration.NewRow(); dmRowConfiguration["BatchDirectory"] = configuration.BatchDirectory; dmRowConfiguration["ConflictResolutionPolicy"] = configuration.ConflictResolutionPolicy; dmRowConfiguration["DownloadBatchSizeInKB"] = configuration.DownloadBatchSizeInKB; dmRowConfiguration["UseBulkOperations"] = configuration.UseBulkOperations; dmRowConfiguration["UseVerboseErrors"] = configuration.UseVerboseErrors; //dmRowConfiguration["OverwriteConfiguration"] = configuration.OverwriteConfiguration; dmRowConfiguration["SerializationConverter"] = configuration.SerializationFormat; dmTableConfiguration.Rows.Add(dmRowConfiguration); if (configuration.ScopeSet != null && configuration.ScopeSet.Tables.Count > 0) { foreach (var dmTable in configuration.ScopeSet.Tables) { var dmTableConf = dmTable.Clone(); set.Tables.Add(dmTableConf); } foreach (var dmRelation in configuration.ScopeSet.Relations) { var dmRelationConf = dmRelation.Clone(set); set.Relations.Add(dmRelationConf); } } if (configuration.Filters.Count > 0) { if (!set.Tables.Contains("DotmimSync__Filters")) { dmTableFilterParameters = new DmTable("DotmimSync__Filters"); set.Tables.Add(dmTableFilterParameters); } dmTableFilterParameters = set.Tables["DotmimSync__Filters"]; dmTableFilterParameters.Columns.Add <String>("TableName"); dmTableFilterParameters.Columns.Add <String>("ColumnName"); foreach (var p in configuration.Filters) { var dmRowFilter = dmTableFilterParameters.NewRow(); dmRowFilter["TableName"] = p.TableName; dmRowFilter["ColumnName"] = p.ColumnName; dmTableFilterParameters.Rows.Add(dmRowFilter); } } }
internal static SyncConfiguration DeserializeFromDmSet(DmSet set) { if (set == null) { return(null); } if (!set.Tables.Contains("DotmimSync__ServiceConfiguration")) { return(null); } SyncConfiguration configuration = new SyncConfiguration(); var dmRowConfiguration = set.Tables["DotmimSync__ServiceConfiguration"].Rows[0]; configuration.BatchDirectory = dmRowConfiguration["BatchDirectory"] as String;; configuration.ConflictResolutionPolicy = (ConflictResolutionPolicy)dmRowConfiguration["ConflictResolutionPolicy"]; configuration.DownloadBatchSizeInKB = (int)dmRowConfiguration["DownloadBatchSizeInKB"]; configuration.UseBulkOperations = (bool)dmRowConfiguration["UseBulkOperations"]; configuration.UseVerboseErrors = (bool)dmRowConfiguration["UseVerboseErrors"]; configuration.OverwriteConfiguration = (bool)dmRowConfiguration["OverwriteConfiguration"]; configuration.SerializationFormat = (SerializationFormat)dmRowConfiguration["SerializationConverter"]; if (set.Tables.Contains("DotmimSync__Filters")) { var dmTableFilterParameters = set.Tables["DotmimSync__Filters"]; foreach (var dmRowFilter in dmTableFilterParameters.Rows) { FilterClause filterClause = new FilterClause(); var tableName = dmRowFilter["TableName"] as String; var columnName = dmRowFilter["ColumnName"] as String; configuration.Filters.Add(tableName, columnName); } } //if (set.Tables.Contains("DotmimSync__Table")) //{ // var dmTableTables = set.Tables["DotmimSync__Table"]; // configuration.Tables = new string[dmTableTables.Rows.Count]; // for (int i = 0; i < dmTableTables.Rows.Count; i++) // { // var dmRowTable = dmTableTables.Rows[i]; // var tableName = dmRowTable["Name"] as String; // configuration.Tables[i] = tableName; // } //} var configTables = set.Tables.Where(tbl => !tbl.TableName.StartsWith("DotmimSync__")); if (configTables != null) { foreach (var configTable in configTables) { configuration.ScopeSet.Tables.Add(configTable.Clone()); } } if (set.Relations != null && set.Relations.Count > 0) { foreach (var r in set.Relations) { var relation = r.Clone(configuration.ScopeSet); configuration.ScopeSet.Relations.Add(relation); } } return(configuration); }