/// <summary> /// Launch a synchronization with the specified mode /// </summary> public async Task <SyncContext> SynchronizeAsync(SyncType syncType, CancellationToken cancellationToken) { // Context, used to back and forth data between servers var context = new SyncContext(Guid.NewGuid()) { // set start time StartTime = DateTime.Now, // if any parameters, set in context Parameters = this.Parameters, // set sync type (Normal, Reinitialize, ReinitializeWithUpload) SyncType = syncType }; this.SessionState = SyncSessionState.Synchronizing; this.SessionStateChanged?.Invoke(this, this.SessionState); ScopeInfo localScopeInfo = null, serverScopeInfo = null, localScopeReferenceInfo = null, scope = null; var fromId = Guid.Empty; var lastSyncTS = 0L; var isNew = true; try { if (cancellationToken.IsCancellationRequested) { cancellationToken.ThrowIfCancellationRequested(); } // Setting the cancellation token this.LocalProvider.SetCancellationToken(cancellationToken); this.RemoteProvider.SetCancellationToken(cancellationToken); if (this.Options == null) { this.Options = new SyncOptions(); } // if local provider does not provider options, get them from sync agent if (this.LocalProvider.Options == null) { this.LocalProvider.Options = this.Options; } if (this.RemoteProvider.Options == null) { this.RemoteProvider.Options = this.Options; } // ---------------------------------------- // 0) Begin Session / Get the Configuration from remote provider // If the configuration object is provided by the client, the server will be updated with it. // ---------------------------------------- (context, this.Configuration) = await this.RemoteProvider.BeginSessionAsync(context, new MessageBeginSession { Configuration = this.Configuration }); if (cancellationToken.IsCancellationRequested) { cancellationToken.ThrowIfCancellationRequested(); } // Locally, nothing really special. Eventually, editing the config object (context, this.Configuration) = await this.LocalProvider.BeginSessionAsync(context, new MessageBeginSession { Configuration = this.Configuration }); if (cancellationToken.IsCancellationRequested) { cancellationToken.ThrowIfCancellationRequested(); } // ---------------------------------------- // 1) Read scope info // ---------------------------------------- // get the scope from local provider List <ScopeInfo> localScopes; List <ScopeInfo> serverScopes; (context, localScopes) = await this.LocalProvider.EnsureScopesAsync(context, new MessageEnsureScopes { ScopeInfoTableName = this.Configuration.ScopeInfoTableName, ScopeName = this.Configuration.ScopeName, SerializationFormat = this.Configuration.SerializationFormat }); if (localScopes.Count != 1) { throw new Exception("On Local provider, we should have only one scope info"); } if (cancellationToken.IsCancellationRequested) { cancellationToken.ThrowIfCancellationRequested(); } localScopeInfo = localScopes[0]; (context, serverScopes) = await this.RemoteProvider.EnsureScopesAsync(context, new MessageEnsureScopes { ScopeInfoTableName = this.Configuration.ScopeInfoTableName, ScopeName = this.Configuration.ScopeName, ClientReferenceId = localScopeInfo.Id, SerializationFormat = this.Configuration.SerializationFormat }); if (serverScopes.Count != 2) { throw new Exception("On Remote provider, we should have two scopes (one for server and one for client side)"); } if (cancellationToken.IsCancellationRequested) { cancellationToken.ThrowIfCancellationRequested(); } serverScopeInfo = serverScopes.First(s => s.Id != localScopeInfo.Id); localScopeReferenceInfo = serverScopes.First(s => s.Id == localScopeInfo.Id); // ---------------------------------------- // 2) Build Configuration Object // ---------------------------------------- // Get Schema from remote provider (context, this.Configuration.Schema) = await this.RemoteProvider.EnsureSchemaAsync(context, new MessageEnsureSchema { Schema = this.Configuration.Schema, SerializationFormat = this.Configuration.SerializationFormat }); if (cancellationToken.IsCancellationRequested) { cancellationToken.ThrowIfCancellationRequested(); } // Apply on local Provider (context, this.Configuration.Schema) = await this.LocalProvider.EnsureSchemaAsync(context, new MessageEnsureSchema { Schema = this.Configuration.Schema, SerializationFormat = this.Configuration.SerializationFormat }); if (cancellationToken.IsCancellationRequested) { cancellationToken.ThrowIfCancellationRequested(); } // ---------------------------------------- // 3) Ensure databases are ready // ---------------------------------------- // Server should have already the schema context = await this.RemoteProvider.EnsureDatabaseAsync(context, new MessageEnsureDatabase { ScopeInfo = serverScopeInfo, Schema = this.Configuration.Schema, Filters = this.Configuration.Filters, SerializationFormat = this.Configuration.SerializationFormat }); if (cancellationToken.IsCancellationRequested) { cancellationToken.ThrowIfCancellationRequested(); } // Client could have, or not, the tables context = await this.LocalProvider.EnsureDatabaseAsync(context, new MessageEnsureDatabase { ScopeInfo = localScopeInfo, Schema = this.Configuration.Schema, Filters = this.Configuration.Filters, SerializationFormat = this.Configuration.SerializationFormat }); if (cancellationToken.IsCancellationRequested) { cancellationToken.ThrowIfCancellationRequested(); } // ---------------------------------------- // 5) Get changes and apply them // ---------------------------------------- BatchInfo clientBatchInfo; BatchInfo serverBatchInfo; ChangesSelected clientChangesSelected = null; ChangesSelected serverChangesSelected = null; ChangesApplied clientChangesApplied = null; ChangesApplied serverChangesApplied = null; // those timestamps will be registered as the "timestamp just before launch the sync" long serverTimestamp, clientTimestamp; if (cancellationToken.IsCancellationRequested) { cancellationToken.ThrowIfCancellationRequested(); } // Apply on the Server Side // Since we are on the server, // we need to check the server client timestamp (not the client timestamp which is completely different) var serverPolicy = this.Configuration.ConflictResolutionPolicy; var clientPolicy = serverPolicy == ConflictResolutionPolicy.ServerWins ? ConflictResolutionPolicy.ClientWins : ConflictResolutionPolicy.ServerWins; // We get from local provider all rows not last updated from the server fromId = serverScopeInfo.Id; // lastSyncTS : get lines inserted / updated / deteleted after the last sync commited lastSyncTS = localScopeInfo.LastSyncTimestamp; // isNew : If isNew, lasttimestamp is not correct, so grab all isNew = localScopeInfo.IsNewScope; //Direction set to Upload context.SyncWay = SyncWay.Upload; // JUST before the whole process, get the timestamp, to be sure to // get rows inserted / updated elsewhere since the sync is not over (context, clientTimestamp) = await this.LocalProvider.GetLocalTimestampAsync(context, new MessageTimestamp { ScopeInfoTableName = this.Configuration.ScopeInfoTableName, SerializationFormat = this.Configuration.SerializationFormat }); if (cancellationToken.IsCancellationRequested) { cancellationToken.ThrowIfCancellationRequested(); } scope = new ScopeInfo { Id = fromId, IsNewScope = isNew, Timestamp = lastSyncTS }; (context, clientBatchInfo, clientChangesSelected) = await this.LocalProvider.GetChangeBatchAsync(context, new MessageGetChangesBatch { ScopeInfo = scope, Schema = this.Configuration.Schema, //BatchSize = this.Options.BatchSize, //BatchDirectory = this.Options.BatchDirectory, Policy = clientPolicy, Filters = this.Configuration.Filters, SerializationFormat = this.Configuration.SerializationFormat }); if (cancellationToken.IsCancellationRequested) { cancellationToken.ThrowIfCancellationRequested(); } // fromId : When applying rows, make sure it's identified as applied by this client scope fromId = localScopeInfo.Id; // lastSyncTS : apply lines only if thye are not modified since last client sync lastSyncTS = localScopeReferenceInfo.LastSyncTimestamp; // isNew : not needed isNew = false; scope = new ScopeInfo { Id = fromId, IsNewScope = isNew, Timestamp = lastSyncTS }; (context, serverChangesApplied) = await this.RemoteProvider.ApplyChangesAsync(context, new MessageApplyChanges { FromScope = scope, Schema = this.Configuration.Schema, Policy = serverPolicy, //UseBulkOperations = this.Options.UseBulkOperations, //CleanMetadatas = this.Options.CleanMetadatas, ScopeInfoTableName = this.Configuration.ScopeInfoTableName, Changes = clientBatchInfo, SerializationFormat = this.Configuration.SerializationFormat }); // if ConflictResolutionPolicy.ClientWins or Handler set to Client wins // Conflict occurs here and server loose. // Conflicts count should be temp saved because applychanges on client side won't raise any conflicts (and so property Context.TotalSyncConflicts will be reset to 0) var conflictsOnRemoteCount = context.TotalSyncConflicts; if (cancellationToken.IsCancellationRequested) { cancellationToken.ThrowIfCancellationRequested(); } // Get changes from server // get the archive if exists //if (localScopeReferenceInfo.IsNewScope && !string.IsNullOrEmpty(this.Configuration.Archive)) //{ //// fromId : Make sure we don't select lines on server that has been already updated by the client //fromId = localScopeInfo.Id; //// lastSyncTS : apply lines only if thye are not modified since last client sync //lastSyncTS = localScopeReferenceInfo.LastTimestamp; //// isNew : make sure we take all lines if it's the first time we get //isNew = localScopeReferenceInfo.IsNewScope; //scope = new ScopeInfo { Id = fromId, IsNewScope = isNew, LastTimestamp = lastSyncTS }; ////Direction set to Download //context.SyncWay = SyncWay.Download; //(context, serverBatchInfo, serverChangesSelected) = await this.RemoteProvider.GetArchiveAsync(context, scope); //// fromId : When applying rows, make sure it's identified as applied by this server scope //fromId = serverScopeInfo.Id; //// lastSyncTS : apply lines only if they are not modified since last client sync //lastSyncTS = localScopeInfo.LastTimestamp; //// isNew : if IsNew, don't apply deleted rows from server //isNew = localScopeInfo.IsNewScope; //scope = new ScopeInfo { Id = fromId, IsNewScope = isNew, LastTimestamp = lastSyncTS }; //(context, clientChangesApplied) = await this.LocalProvider.ApplyArchiveAsync(context, scope, serverBatchInfo); //// Here we have to change the localScopeInfo.LastTimestamp to the good one //// last ts from archive //localScopeReferenceInfo.LastTimestamp = [something from the archive]; //// we are not new anymore //localScopeReferenceInfo.IsNewScope = false; //} // fromId : Make sure we don't select lines on server that has been already updated by the client fromId = localScopeInfo.Id; // lastSyncTS : apply lines only if thye are not modified since last client sync lastSyncTS = localScopeReferenceInfo.LastSyncTimestamp; // isNew : make sure we take all lines if it's the first time we get isNew = localScopeReferenceInfo.IsNewScope; scope = new ScopeInfo { Id = fromId, IsNewScope = isNew, Timestamp = lastSyncTS }; //Direction set to Download context.SyncWay = SyncWay.Download; // JUST Before get changes, get the timestamp, to be sure to // get rows inserted / updated elsewhere since the sync is not over (context, serverTimestamp) = await this.RemoteProvider.GetLocalTimestampAsync(context, new MessageTimestamp { ScopeInfoTableName = this.Configuration.ScopeInfoTableName, SerializationFormat = this.Configuration.SerializationFormat }); if (cancellationToken.IsCancellationRequested) { cancellationToken.ThrowIfCancellationRequested(); } (context, serverBatchInfo, serverChangesSelected) = await this.RemoteProvider.GetChangeBatchAsync(context, new MessageGetChangesBatch { ScopeInfo = scope, Schema = this.Configuration.Schema, //BatchSize = this.Options.BatchSize, //BatchDirectory = this.Options.BatchDirectory, Policy = serverPolicy, Filters = this.Configuration.Filters, SerializationFormat = this.Configuration.SerializationFormat }); if (cancellationToken.IsCancellationRequested) { cancellationToken.ThrowIfCancellationRequested(); } // Apply local changes // fromId : When applying rows, make sure it's identified as applied by this server scope fromId = serverScopeInfo.Id; // lastSyncTS : apply lines only if they are not modified since last client sync lastSyncTS = localScopeInfo.LastSyncTimestamp; // isNew : if IsNew, don't apply deleted rows from server isNew = localScopeInfo.IsNewScope; scope = new ScopeInfo { Id = fromId, IsNewScope = isNew, Timestamp = lastSyncTS }; (context, clientChangesApplied) = await this.LocalProvider.ApplyChangesAsync(context, new MessageApplyChanges { FromScope = scope, Schema = this.Configuration.Schema, Policy = clientPolicy, //UseBulkOperations = this.Options.UseBulkOperations, //CleanMetadatas = this.Options.CleanMetadatas, ScopeInfoTableName = this.Configuration.ScopeInfoTableName, Changes = serverBatchInfo, SerializationFormat = this.Configuration.SerializationFormat }); context.TotalChangesDownloaded = clientChangesApplied.TotalAppliedChanges; context.TotalChangesUploaded = clientChangesSelected.TotalChangesSelected; context.TotalSyncErrors = clientChangesApplied.TotalAppliedChangesFailed; context.CompleteTime = DateTime.Now; serverScopeInfo.IsNewScope = false; localScopeReferenceInfo.IsNewScope = false; localScopeInfo.IsNewScope = false; serverScopeInfo.LastSync = context.CompleteTime; localScopeReferenceInfo.LastSync = context.CompleteTime; localScopeInfo.LastSync = context.CompleteTime; serverScopeInfo.LastSyncTimestamp = serverTimestamp; localScopeReferenceInfo.LastSyncTimestamp = serverTimestamp; localScopeInfo.LastSyncTimestamp = clientTimestamp; var duration = context.CompleteTime.Subtract(context.StartTime); serverScopeInfo.LastSyncDuration = duration.Ticks; localScopeReferenceInfo.LastSyncDuration = duration.Ticks; localScopeInfo.LastSyncDuration = duration.Ticks; serverScopeInfo.IsLocal = true; localScopeReferenceInfo.IsLocal = false; context = await this.RemoteProvider.WriteScopesAsync(context, new MessageWriteScopes { ScopeInfoTableName = this.Configuration.ScopeInfoTableName, Scopes = new List <ScopeInfo> { serverScopeInfo, localScopeReferenceInfo }, SerializationFormat = this.Configuration.SerializationFormat }); serverScopeInfo.IsLocal = false; localScopeInfo.IsLocal = true; if (cancellationToken.IsCancellationRequested) { cancellationToken.ThrowIfCancellationRequested(); } context = await this.LocalProvider.WriteScopesAsync(context, new MessageWriteScopes { ScopeInfoTableName = this.Configuration.ScopeInfoTableName, Scopes = new List <ScopeInfo> { localScopeInfo, serverScopeInfo }, SerializationFormat = this.Configuration.SerializationFormat }); if (cancellationToken.IsCancellationRequested) { cancellationToken.ThrowIfCancellationRequested(); } } catch (SyncException se) { Console.WriteLine($"Sync Exception: {se.Message}. Type:{se.Type}. On provider: {se.ProviderName}."); throw; } catch (Exception ex) { Console.WriteLine($"Unknwon Exception: {ex.Message}."); throw new SyncException(ex, SyncStage.None, string.Empty); } finally { // End the current session context = await this.RemoteProvider.EndSessionAsync(context); context = await this.LocalProvider.EndSessionAsync(context); this.SessionState = SyncSessionState.Ready; this.SessionStateChanged?.Invoke(this, this.SessionState); } return(context); }
public MigratedArgs(SyncContext context, SyncSet schema, SyncSetup setup, MigrationResults migration, DbConnection connection = null, DbTransaction transaction = null) : base(context, connection, transaction) { this.Schema = schema; this.Setup = setup; this.Migration = migration; }
public ProgressArgs(SyncContext context, string message, DbConnection connection, DbTransaction transaction) : this(context, connection, transaction) => this.Message = message;
public ClientScopeInfoLoadedArgs(SyncContext context, string scopeName, ClientScopeInfo clientScopeInfo, DbConnection connection = null, DbTransaction transaction = null) : base(context, connection, transaction) { this.ScopeName = scopeName; this.ClientScopeInfo = clientScopeInfo; }
public ServerScopeInfoLoadingArgs(SyncContext context, string scopeName, DbCommand command, DbConnection connection = null, DbTransaction transaction = null) : base(context, connection, transaction) { this.Command = command; this.ScopeName = scopeName; }
public Task MigrateAsync(SyncContext context) { return(null); //DbTransaction transaction = null; //using (var connection = this.provider.CreateConnection()) //{ // // Encapsulate in a try catch for a better exception handling // // Especially whe called from web proxy // try // { // await connection.OpenAsync().ConfigureAwait(false); // // Let provider knows a connection is opened // this.provider.OnConnectionOpened(connection); // await this.provider.InterceptAsync(new ConnectionOpenArgs(context, connection)).ConfigureAwait(false); // // Create a transaction // using (transaction = connection.BeginTransaction(this.provider.IsolationLevel)) // { // await this.provider.InterceptAsync(new TransactionOpenArgs(context, connection, transaction)).ConfigureAwait(false); // // actual scope info table name // var scopeInfoTableName = string.IsNullOrEmpty(this.currentScopeInfoTableName) ? this.newOptions.ScopeInfoTableName : this.currentScopeInfoTableName; // // create a temp sync context // ScopeInfo localScope = null; // var scopeBuilder = this.provider.GetScopeBuilder(); // var scopeInfoBuilder = scopeBuilder.CreateScopeInfoBuilder(scopeInfoTableName, connection, transaction); // // if current scope table name does not exists, it's probably first sync. so return // if (scopeInfoBuilder.NeedToCreateClientScopeInfoTable()) // return; // // Get scope // (context, localScope) = await this.provider.GetClientScopeAsync( // context, this.newOptions.ScopeName, // connection, transaction, CancellationToken.None).ConfigureAwait(false); // // Get current schema saved in local database // if (localScope == null || string.IsNullOrEmpty(localScope.Schema)) // return; // var currentSchema = JsonConvert.DeserializeObject<SyncSet>(localScope.Schema); // // Create new schema based on new setup // var newSchema = this.provider.ReadSchema(this.newSetup, connection, transaction); // // Get Tables that are NOT in new schema anymore // var deletedSyncTables = currentSchema.Tables.Where(currentTable => !newSchema.Tables.Any(newTable => newTable == currentTable)); // foreach (var dSyncTable in deletedSyncTables) // { // // get builder // var delBuilder = this.provider.GetTableBuilder(dSyncTable); // // Delete all stored procedures // delBuilder.DropProcedures(connection, transaction); // // Delete all triggers // delBuilder.DropTriggers(connection, transaction); // // Delete tracking table // delBuilder.DropTrackingTable(connection, transaction); // } // // Get Tables that are completely new // var addSyncTables = newSchema.Tables.Where(newTable => !currentSchema.Tables.Any(currentTable => newTable == currentTable)); // foreach (var aSyncTable in addSyncTables) // { // // get builder // var addBuilder = this.provider.GetTableBuilder(aSyncTable); // // Create table if not exists // addBuilder.CreateTable(connection, transaction); // // Create tracking table // addBuilder.CreateTrackingTable(connection, transaction); // // Create triggers // addBuilder.CreateTriggers(connection, transaction); // // Create stored procedures // addBuilder.CreateStoredProcedures(connection, transaction); // } // var editSyncTables = newSchema.Tables.Where(newTable => currentSchema.Tables.Any(currentTable => newTable == currentTable)); // foreach (var eSyncTable in editSyncTables) // { // var cSyncTable = currentSchema.Tables.First(t => t == eSyncTable); // var migrationTable = new DbMigrationTable(this.provider, cSyncTable, eSyncTable, true); // migrationTable.Compare(); // //migrationTable.Apply(connection, transaction); // } // await this.provider.InterceptAsync(new TransactionCommitArgs(null, connection, transaction)).ConfigureAwait(false); // transaction.Commit(); // } // } // catch (Exception ex) // { // var syncException = new SyncException(ex, context.SyncStage); // // try to let the provider enrich the exception // this.provider.EnsureSyncException(syncException); // syncException.Side = SyncExceptionSide.ClientSide; // throw syncException; // } // finally // { // if (transaction != null) // transaction.Dispose(); // if (connection != null && connection.State == ConnectionState.Open) // connection.Close(); // await this.provider.InterceptAsync(new ConnectionCloseArgs(context, connection, transaction)).ConfigureAwait(false); // // Let provider knows a connection is closed // this.provider.OnConnectionClosed(connection); // } //} }
/// <summary> /// Deprovision a database. You have to passe a configuration object, containing at least the dmTables /// </summary> public async Task <SyncContext> DeprovisionAsync(SyncContext context, SyncSet schema, SyncSetup setup, SyncProvision provision, string scopeInfoTableName, bool disableConstraintsOnApplyChanges, DbConnection connection, DbTransaction transaction, CancellationToken cancellationToken, IProgress <ProgressArgs> progress) { if (schema.Tables == null || !schema.HasTables) { throw new MissingTablesException(); } this.Orchestrator.logger.LogDebug(SyncEventsId.Deprovision, new { TablesCount = schema.Tables.Count, ScopeInfoTableName = scopeInfoTableName, DisableConstraintsOnApplyChanges = disableConstraintsOnApplyChanges, }); // get Database builder var builder = this.GetDatabaseBuilder(); builder.UseChangeTracking = this.UseChangeTracking; builder.UseBulkProcedures = this.SupportBulkOperations; // Sorting tables based on dependencies between them var schemaTables = schema.Tables .SortByDependencies(tab => tab.GetRelations() .Select(r => r.GetParentTable())); // Disable check constraints if (disableConstraintsOnApplyChanges) { foreach (var table in schemaTables.Reverse()) { await this.DisableConstraintsAsync(context, table, setup, connection, transaction).ConfigureAwait(false); } } // Creating a local function to mutualize call var deprovisionFuncAsync = new Func <SyncProvision, IEnumerable <SyncTable>, Task>(async(p, tables) => { foreach (var schemaTable in tables) { var tableBuilder = this.GetTableBuilder(schemaTable, setup); // set if the builder supports creating the bulk operations proc stock tableBuilder.UseBulkProcedures = this.SupportBulkOperations; tableBuilder.UseChangeTracking = this.UseChangeTracking; // adding filter this.AddFilters(schemaTable, tableBuilder); this.Orchestrator.logger.LogDebug(SyncEventsId.Deprovision, schemaTable); await tableBuilder.DropAsync(p, connection, transaction).ConfigureAwait(false); // Interceptor await this.Orchestrator.InterceptAsync(new TableDeprovisionedArgs(context, p, schemaTable, connection, transaction), cancellationToken).ConfigureAwait(false); } }); // Checking if we have to deprovision tables bool hasDeprovisionTableFlag = provision.HasFlag(SyncProvision.Table); // Firstly, removing the flag from the provision, because we need to drop everything in correct order, then drop tables in reverse side if (hasDeprovisionTableFlag) { provision ^= SyncProvision.Table; } // Deprovision everything in order, excepting table await deprovisionFuncAsync(provision, schemaTables).ConfigureAwait(false); // then in reverse side, deprovision tables, if Table was part of Provision enumeration. if (hasDeprovisionTableFlag) { await deprovisionFuncAsync(SyncProvision.Table, schemaTables.Reverse()).ConfigureAwait(false); } if (provision.HasFlag(SyncProvision.ClientScope)) { context = await this.DropClientScopeAsync(context, scopeInfoTableName, connection, transaction, cancellationToken, progress).ConfigureAwait(false); } if (provision.HasFlag(SyncProvision.ServerScope)) { context = await this.DropServerScopeAsync(context, scopeInfoTableName, connection, transaction, cancellationToken, progress).ConfigureAwait(false); } if (provision.HasFlag(SyncProvision.ServerHistoryScope)) { context = await this.DropServerHistoryScopeAsync(context, scopeInfoTableName, connection, transaction, cancellationToken, progress).ConfigureAwait(false); } return(context); }
/// <summary> /// Launch a synchronization with the specified mode /// </summary> public async Task <SyncResult> SynchronizeAsync(string scopeName, SyncSetup setup, SyncType syncType, SyncParameters parameters, CancellationToken cancellationToken, IProgress <ProgressArgs> progress = null) { // checkpoints dates var startTime = DateTime.UtcNow; var completeTime = DateTime.UtcNow; // Create a logger var logger = this.Options.Logger ?? new SyncLogger().AddDebug(); // Lock sync to prevent multi call to sync at the same time LockSync(); // Context, used to back and forth data between servers var context = new SyncContext(Guid.NewGuid(), scopeName) { // if any parameters, set in context Parameters = parameters, // set sync type (Normal, Reinitialize, ReinitializeWithUpload) SyncType = syncType }; // Result, with sync results stats. var result = new SyncResult(context.SessionId) { // set start time StartTime = startTime, CompleteTime = completeTime, }; this.SessionState = SyncSessionState.Synchronizing; this.SessionStateChanged?.Invoke(this, this.SessionState); //await Task.Run(async () => //{ try { if (cancellationToken.IsCancellationRequested) { cancellationToken.ThrowIfCancellationRequested(); } if (setup != null) { var remoteOrchestratorType = this.RemoteOrchestrator.GetType(); var providerType = remoteOrchestratorType.Name; if (providerType.ToLowerInvariant() == "webclientorchestrator" || providerType.ToLowerInvariant() == "webremotetorchestrator") { throw new Exception("Do not set Tables (or SyncSetup) from your client. Please use SyncAgent, without any Tables or SyncSetup. The tables will come from the server side"); } } // Begin session context = await this.LocalOrchestrator.InternalBeginSessionAsync(context, cancellationToken, progress).ConfigureAwait(false); if (cancellationToken.IsCancellationRequested) { cancellationToken.ThrowIfCancellationRequested(); } // no need to check on every call to SynchronizeAsync if (!checkUpgradeDone) { var needToUpgrade = await this.LocalOrchestrator.NeedsToUpgradeAsync(default, default, cancellationToken, progress).ConfigureAwait(false);
/// <summary> /// Be sure all tables are ready and configured for sync /// the ScopeSet Configuration MUST be filled by the schema form Database /// </summary> public virtual async Task <SyncContext> EnsureDatabaseAsync(SyncContext context, MessageEnsureDatabase message) { DbConnection connection = null; try { // Event progress context.SyncStage = SyncStage.DatabaseApplying; DatabaseApplyingEventArgs beforeArgs = new DatabaseApplyingEventArgs(this.ProviderTypeName, context.SyncStage, message.Schema); this.TryRaiseProgressEvent(beforeArgs, this.DatabaseApplying); // If scope exists and lastdatetime sync is present, so database exists // Check if we don't have an OverwriteConfiguration (if true, we force the check) if (message.ScopeInfo.LastSync.HasValue && !beforeArgs.OverwriteSchema) { return(context); } StringBuilder script = new StringBuilder(); // Open the connection using (connection = this.CreateConnection()) { await connection.OpenAsync(); using (var transaction = connection.BeginTransaction()) { // Sorting tables based on dependencies between them var dmTables = message.Schema.Tables .SortByDependencies(tab => tab.ChildRelations .Select(r => r.ChildTable)); foreach (var dmTable in dmTables) { var builder = GetDatabaseBuilder(dmTable); // set if the builder supports creating the bulk operations proc stock builder.UseBulkProcedures = this.SupportBulkOperations; // adding filter this.AddFilters(message.Filters, dmTable, builder); context.SyncStage = SyncStage.DatabaseTableApplying; DatabaseTableApplyingEventArgs beforeTableArgs = new DatabaseTableApplyingEventArgs(this.ProviderTypeName, context.SyncStage, dmTable.TableName); this.TryRaiseProgressEvent(beforeTableArgs, this.DatabaseTableApplying); string currentScript = null; if (beforeArgs.GenerateScript) { currentScript = builder.ScriptTable(connection, transaction); currentScript += builder.ScriptForeignKeys(connection, transaction); script.Append(currentScript); } builder.Create(connection, transaction); builder.CreateForeignKeys(connection, transaction); context.SyncStage = SyncStage.DatabaseTableApplied; DatabaseTableAppliedEventArgs afterTableArgs = new DatabaseTableAppliedEventArgs(this.ProviderTypeName, context.SyncStage, dmTable.TableName, currentScript); this.TryRaiseProgressEvent(afterTableArgs, this.DatabaseTableApplied); } context.SyncStage = SyncStage.DatabaseApplied; var afterArgs = new DatabaseAppliedEventArgs(this.ProviderTypeName, context.SyncStage, script.ToString()); this.TryRaiseProgressEvent(afterArgs, this.DatabaseApplied); transaction.Commit(); } connection.Close(); return(context); } } catch (Exception ex) { throw new SyncException(ex, SyncStage.DatabaseApplying, this.ProviderTypeName); } finally { if (connection != null && connection.State != ConnectionState.Closed) { connection.Close(); } } }
public HttpGettingResponseMessageArgs(HttpResponseMessage response, SyncContext context) : base(context, null, null) { this.Response = response; }
public HttpSendingRequestMessageArgs(HttpRequestMessage request, SyncContext context) : base(context, null, null) { this.Request = request; }
/// <summary> /// Enumerate all internal changes, no batch mode /// </summary> internal async Task <(BatchInfo, ChangesSelected)> EnumerateChangesInBatchesInternal(SyncContext context, ScopeInfo scopeInfo) { Debug.WriteLine($"----- Enumerating Changes for Scope \"{scopeInfo.Name}\" -----"); Debug.WriteLine(""); Debug.WriteLine(""); var configuration = GetCacheConfiguration(); // memory size total double memorySizeFromDmRows = 0L; int batchIndex = 0; // this batch info won't be in memory, it will be be batched BatchInfo batchInfo = new BatchInfo(); // directory where all files will be stored batchInfo.Directory = BatchInfo.GenerateNewDirectoryName(); // not in memory since we serialized all files in the tmp directory batchInfo.InMemory = false; // Create stats object to store changes count ChangesSelected changes = new ChangesSelected(); using (var connection = this.CreateConnection()) { try { // Open the connection await connection.OpenAsync(); using (var transaction = connection.BeginTransaction()) { // create the in memory changes set DmSet changesSet = new DmSet(configuration.ScopeSet.DmSetName); foreach (var tableDescription in configuration) { // if we are in upload stage, so check if table is not download only if (context.SyncWay == SyncWay.Upload && tableDescription.SyncDirection == SyncDirection.DownloadOnly) { continue; } // if we are in download stage, so check if table is not download only if (context.SyncWay == SyncWay.Download && tableDescription.SyncDirection == SyncDirection.UploadOnly) { continue; } var builder = this.GetDatabaseBuilder(tableDescription); var syncAdapter = builder.CreateSyncAdapter(connection, transaction); syncAdapter.ConflictApplyAction = configuration.GetApplyAction(); // raise before event context.SyncStage = SyncStage.TableChangesSelecting; var beforeArgs = new TableChangesSelectingEventArgs(this.ProviderTypeName, context.SyncStage, tableDescription.TableName); this.TryRaiseProgressEvent(beforeArgs, this.TableChangesSelecting); // Get Command DbCommand selectIncrementalChangesCommand; DbCommandType dbCommandType; if (this.CanBeServerProvider && context.Parameters != null && context.Parameters.Count > 0 && configuration.Filters != null && configuration.Filters.Count > 0) { var filtersName = configuration.Filters .Where(f => f.TableName.Equals(tableDescription.TableName, StringComparison.InvariantCultureIgnoreCase)) .Select(f => f.ColumnName); if (filtersName != null && filtersName.Count() > 0) { dbCommandType = DbCommandType.SelectChangesWitFilters; selectIncrementalChangesCommand = syncAdapter.GetCommand(dbCommandType, filtersName); } else { dbCommandType = DbCommandType.SelectChanges; selectIncrementalChangesCommand = syncAdapter.GetCommand(dbCommandType); } } else { dbCommandType = DbCommandType.SelectChanges; selectIncrementalChangesCommand = syncAdapter.GetCommand(dbCommandType); } // Deriving Parameters syncAdapter.SetCommandParameters(DbCommandType.SelectChanges, selectIncrementalChangesCommand); if (selectIncrementalChangesCommand == null) { var exc = "Missing command 'SelectIncrementalChangesCommand' "; Debug.WriteLine(exc); throw new Exception(exc); } var dmTable = BuildChangesTable(tableDescription.TableName); try { // Set commons parameters SetSelectChangesCommonParameters(context, scopeInfo, selectIncrementalChangesCommand); // Set filter parameters if any // Only on server side if (this.CanBeServerProvider && context.Parameters != null && context.Parameters.Count > 0 && configuration.Filters != null && configuration.Filters.Count > 0) { var filters = configuration.Filters.Where(f => f.TableName.Equals(tableDescription.TableName, StringComparison.InvariantCultureIgnoreCase)).ToList(); if (filters != null && filters.Count > 0) { foreach (var filter in filters) { var parameter = context.Parameters.FirstOrDefault(p => p.ColumnName.Equals(filter.ColumnName, StringComparison.InvariantCultureIgnoreCase) && p.TableName.Equals(filter.TableName, StringComparison.InvariantCultureIgnoreCase)); if (parameter != null) { DbManager.SetParameterValue(selectIncrementalChangesCommand, parameter.ColumnName, parameter.Value); } } } } this.AddTrackingColumns <int>(dmTable, "sync_row_is_tombstone"); // Statistics TableChangesSelected tableChangesSelected = new TableChangesSelected { TableName = tableDescription.TableName }; changes.TableChangesSelected.Add(tableChangesSelected); // Get the reader using (var dataReader = selectIncrementalChangesCommand.ExecuteReader()) { while (dataReader.Read()) { DmRow dmRow = CreateRowFromReader(dataReader, dmTable); DmRowState state = DmRowState.Unchanged; state = GetStateFromDmRow(dmRow, scopeInfo); // If the row is not deleted inserted or modified, go next if (state != DmRowState.Deleted && state != DmRowState.Modified && state != DmRowState.Added) { continue; } var fieldsSize = DmTableSurrogate.GetRowSizeFromDataRow(dmRow); var dmRowSize = fieldsSize / 1024d; if (dmRowSize > configuration.DownloadBatchSizeInKB) { var exc = $"Row is too big ({dmRowSize} kb.) for the current Configuration.DownloadBatchSizeInKB ({configuration.DownloadBatchSizeInKB} kb.) Aborting Sync..."; Debug.WriteLine(exc); throw new Exception(exc); } // Calculate the new memory size memorySizeFromDmRows = memorySizeFromDmRows + dmRowSize; // add row dmTable.Rows.Add(dmRow); tableChangesSelected.TotalChanges++; // acceptchanges before modifying dmRow.AcceptChanges(); // Set the correct state to be applied if (state == DmRowState.Deleted) { dmRow.Delete(); tableChangesSelected.Deletes++; } else if (state == DmRowState.Added) { dmRow.SetAdded(); tableChangesSelected.Inserts++; } else if (state == DmRowState.Modified) { dmRow.SetModified(); tableChangesSelected.Updates++; } // We exceed the memorySize, so we can add it to a batch if (memorySizeFromDmRows > configuration.DownloadBatchSizeInKB) { // Since we dont need this column anymore, remove it this.RemoveTrackingColumns(dmTable, "sync_row_is_tombstone"); changesSet.Tables.Add(dmTable); // generate the batch part info batchInfo.GenerateBatchInfo(batchIndex, changesSet, configuration.BatchDirectory); // increment batch index batchIndex++; changesSet.Clear(); // Recreate an empty DmSet, then a dmTable clone changesSet = new DmSet(configuration.ScopeSet.DmSetName); dmTable = dmTable.Clone(); this.AddTrackingColumns <int>(dmTable, "sync_row_is_tombstone"); // Init the row memory size memorySizeFromDmRows = 0L; //// raise SyncProgress Event //// in batch mode, we could have a table on mulitple batchs //// so try to get it //var existSelectedChanges = changes.TableChangesSelected.FirstOrDefault(sc => string.Equals(sc.TableName, tableDescription.TableName)); //if (existSelectedChanges == null) //{ // existSelectedChanges = tableChangesSelected; // changes.TableChangesSelected.Add(tableChangesSelected); //} //else //{ // existSelectedChanges.Deletes += tableChangesSelected.Deletes; // existSelectedChanges.Inserts += tableChangesSelected.Inserts; // existSelectedChanges.Updates += tableChangesSelected.Updates; // existSelectedChanges.TotalChanges += tableChangesSelected.TotalChanges; //} // add stats for a SyncProgress event context.SyncStage = SyncStage.TableChangesSelected; var args2 = new TableChangesSelectedEventArgs (this.ProviderTypeName, SyncStage.TableChangesSelected, tableChangesSelected); this.TryRaiseProgressEvent(args2, this.TableChangesSelected); //// reinit stats //tableChangesSelected = new TableChangesSelected(); //tableChangesSelected.TableName = tableDescription.TableName; } } // Since we dont need this column anymore, remove it this.RemoveTrackingColumns(dmTable, "sync_row_is_tombstone"); context.SyncStage = SyncStage.TableChangesSelected; changesSet.Tables.Add(dmTable); // Init the row memory size memorySizeFromDmRows = 0L; //// raise SyncProgress Event //var esc = changes.TableChangesSelected.FirstOrDefault(sc => string.Equals(sc.TableName, tableDescription.TableName)); //if (esc == null) //{ // esc = tableChangesSelected; // changes.TableChangesSelected.Add(esc); //} //else //{ // esc.Deletes += tableChangesSelected.Deletes; // esc.Inserts += tableChangesSelected.Inserts; // esc.Updates += tableChangesSelected.Updates; // esc.TotalChanges += tableChangesSelected.TotalChanges; //} // Event progress context.SyncStage = SyncStage.TableChangesSelected; var args = new TableChangesSelectedEventArgs(this.ProviderTypeName, SyncStage.TableChangesSelected, tableChangesSelected); this.TryRaiseProgressEvent(args, this.TableChangesSelected); } } catch (Exception dbException) { Debug.WriteLine($"Caught exception while enumerating changes\n{dbException}\n"); throw; } finally { Debug.WriteLine($"--- End Table \"{tableDescription.TableName}\" ---"); Debug.WriteLine(""); } } // We are in batch mode, and we are at the last batchpart info var batchPartInfo = batchInfo.GenerateBatchInfo(batchIndex, changesSet, configuration.BatchDirectory); if (batchPartInfo != null) { batchPartInfo.IsLastBatch = true; } transaction.Commit(); } } catch (Exception) { throw; } finally { if (connection != null && connection.State == ConnectionState.Open) { connection.Close(); } } } Debug.WriteLine($"--- End Enumerating Changes for Scope \"{scopeInfo.Name}\" ---"); Debug.WriteLine(""); return(batchInfo, changes); }
/// <summary> /// Gets a batch of changes to synchronize when given batch size, /// destination knowledge, and change data retriever parameters. /// </summary> /// <returns>A DbSyncContext object that will be used to retrieve the modified data.</returns> public virtual async Task <(SyncContext, BatchInfo, ChangesSelected)> GetChangeBatchAsync(SyncContext context, ScopeInfo scopeInfo) { try { if (scopeInfo == null) { throw new ArgumentNullException("scopeInfo", "Client scope info is null"); } var configuration = GetCacheConfiguration(); // Check if the provider is not outdated var isOutdated = this.IsRemoteOutdated(); // Get a chance to make the sync even if it's outdated if (isOutdated) { var outdatedEventArgs = new OutdatedEventArgs(); this.SyncOutdated?.Invoke(this, outdatedEventArgs); if (outdatedEventArgs.Action != OutdatedSyncAction.Rollback) { context.SyncType = outdatedEventArgs.Action == OutdatedSyncAction.Reinitialize ? SyncType.Reinitialize : SyncType.ReinitializeWithUpload; } if (outdatedEventArgs.Action == OutdatedSyncAction.Rollback) { throw new OutOfDateException("The provider is out of date ! Try to make a Reinitialize sync"); } } // batch info containing changes BatchInfo batchInfo; // Statistics about changes that are selected ChangesSelected changesSelected; // if we try a Reinitialize action, don't get any changes from client // else get changes from batch or in memory methods if (context.SyncWay == SyncWay.Upload && context.SyncType == SyncType.Reinitialize) { (batchInfo, changesSelected) = this.GetEmptyChanges(context, scopeInfo); } else if (configuration.DownloadBatchSizeInKB == 0) { (batchInfo, changesSelected) = await this.EnumerateChangesInternal(context, scopeInfo); } else { (batchInfo, changesSelected) = await this.EnumerateChangesInBatchesInternal(context, scopeInfo); } return(context, batchInfo, changesSelected); } catch (Exception ex) { throw new SyncException(ex, SyncStage.TableChangesSelecting, this.ProviderTypeName); } }
public virtual async Task <(SyncContext SyncContext, string DatabaseName, string Version)> GetHelloAsync(SyncContext context, DbConnection connection, DbTransaction transaction, CancellationToken cancellationToken, IProgress <ProgressArgs> progress) { // get database builder var databaseBuilder = this.GetDatabaseBuilder(); var hello = await databaseBuilder.GetHelloAsync(connection, transaction); this.Orchestrator.logger.LogDebug(SyncEventsId.GetHello, new { hello.DatabaseName, hello.Version }); return(context, hello.DatabaseName, hello.Version); }
/// <summary> /// Called when the sync ensure scopes are created /// </summary> public virtual async Task <(SyncContext, List <ScopeInfo>)> EnsureScopesAsync(SyncContext context, string scopeName, Guid?clientReferenceId = null) { DbConnection connection = null; try { if (string.IsNullOrEmpty(scopeName)) { throw new ArgumentNullException("ScopeName", "Scope name is required"); } context.SyncStage = SyncStage.ScopeLoading; List <ScopeInfo> scopes = new List <ScopeInfo>(); // Open the connection using (connection = this.CreateConnection()) { await connection.OpenAsync(); using (var transaction = connection.BeginTransaction()) { var scopeBuilder = this.GetScopeBuilder(); var scopeInfoBuilder = scopeBuilder.CreateScopeInfoBuilder(connection, transaction); var needToCreateScopeInfoTable = scopeInfoBuilder.NeedToCreateScopeInfoTable(); // create the scope info table if needed if (needToCreateScopeInfoTable) { scopeInfoBuilder.CreateScopeInfoTable(); } // not the first time we ensure scopes, so get scopes if (!needToCreateScopeInfoTable) { // get all scopes shared by all (identified by scopeName) var lstScopes = scopeInfoBuilder.GetAllScopes(scopeName); // try to get the scopes from database // could be two scopes if from server or a single scope if from client scopes = lstScopes.Where(s => (s.IsLocal == true || (clientReferenceId.HasValue && s.Id == clientReferenceId.Value))).ToList(); } // If no scope found, create it on the local provider if (scopes == null || scopes.Count <= 0) { scopes = new List <ScopeInfo>(); // create a new scope id for the current owner (could be server or client as well) var scope = new ScopeInfo(); scope.Id = Guid.NewGuid(); scope.Name = scopeName; scope.IsLocal = true; scope.IsNewScope = true; scope.LastSync = null; scope = scopeInfoBuilder.InsertOrUpdateScopeInfo(scope); scopes.Add(scope); } else { //check if we have alread a good last sync. if no, treat it as new scopes.ForEach(sc => sc.IsNewScope = sc.LastSync == null); } // if we are not on the server, we have to check that we only have one scope if (!clientReferenceId.HasValue && scopes.Count > 1) { throw new InvalidOperationException("On Local provider, we should have only one scope info"); } // if we have a reference in args, we need to get this specific line from database // this happen only on the server side if (clientReferenceId.HasValue) { var refScope = scopes.FirstOrDefault(s => s.Id == clientReferenceId); if (refScope == null) { refScope = new ScopeInfo(); refScope.Id = clientReferenceId.Value; refScope.Name = scopeName; refScope.IsLocal = false; refScope.IsNewScope = true; refScope.LastSync = null; refScope = scopeInfoBuilder.InsertOrUpdateScopeInfo(refScope); scopes.Add(refScope); } else { refScope.IsNewScope = refScope.LastSync == null; } } transaction.Commit(); } connection.Close(); } // Event progress this.TryRaiseProgressEvent( new ScopeEventArgs(this.ProviderTypeName, context.SyncStage, scopes.FirstOrDefault(s => s.IsLocal)), ScopeLoading); return(context, scopes); } catch (Exception ex) { throw new SyncException(ex, SyncStage.ScopeLoading, this.ProviderTypeName); } finally { if (connection != null && connection.State != ConnectionState.Closed) { connection.Close(); } } }
/// <summary> /// Check /// </summary> public virtual async Task <(SyncContext, bool, ServerScopeInfo)> IsConflictingSetupAsync(SyncContext context, SyncSetup inputSetup, ServerScopeInfo serverScopeInfo, DbConnection connection = default, DbTransaction transaction = default, CancellationToken cancellationToken = default, IProgress <ProgressArgs> progress = null) { if (serverScopeInfo.IsNewScope || serverScopeInfo.Schema == null) { return(context, false, serverScopeInfo); } if (inputSetup != null && serverScopeInfo.Setup != null && !serverScopeInfo.Setup.EqualsByProperties(inputSetup)) { var conflictingSetupArgs = new ConflictingSetupArgs(context, inputSetup, null, serverScopeInfo); await this.InterceptAsync(conflictingSetupArgs, progress, cancellationToken).ConfigureAwait(false); if (conflictingSetupArgs.Action == ConflictingSetupAction.Rollback) { throw new Exception("Seems you are trying another Setup tables that what is stored in your server scope database. Please create a new scope or deprovision and provision again your server scope."); } if (conflictingSetupArgs.Action == ConflictingSetupAction.Abort) { return(context, true, serverScopeInfo); } // re affect scope infos serverScopeInfo = conflictingSetupArgs.ServerScopeInfo; } // We gave 2 chances to user to edit the setup and fill correct values. // Final check, but if not valid, raise an error if (inputSetup != null && serverScopeInfo.Setup != null && !serverScopeInfo.Setup.EqualsByProperties(inputSetup)) { throw new Exception("Seems you are trying another Setup tables that what is stored in your server scope database. Please make a migration or create a new scope"); } return(context, false, serverScopeInfo); }
public StoredProcedureDroppedArgs(SyncContext context, SyncTable table, DbStoredProcedureType StoredProcedureType, DbConnection connection = null, DbTransaction transaction = null) : base(context, connection, transaction) { Table = table; this.StoredProcedureType = StoredProcedureType; }
InternalGetServerScopeInfoAsync(SyncContext context, SyncSetup setup, bool overwrite, DbConnection connection, DbTransaction transaction, CancellationToken cancellationToken, IProgress <ProgressArgs> progress) { try { await using var runner = await this.GetConnectionAsync(context, SyncMode.Writing, SyncStage.ScopeLoading, connection, transaction, cancellationToken, progress).ConfigureAwait(false); bool exists; (context, exists) = await this.InternalExistsScopeInfoTableAsync(context, DbScopeType.Server, runner.Connection, runner.Transaction, runner.CancellationToken, runner.Progress).ConfigureAwait(false); if (!exists) { await this.InternalCreateScopeInfoTableAsync(context, DbScopeType.Server, runner.Connection, runner.Transaction, runner.CancellationToken, runner.Progress).ConfigureAwait(false); } (context, exists) = await this.InternalExistsScopeInfoTableAsync(context, DbScopeType.ServerHistory, runner.Connection, runner.Transaction, runner.CancellationToken, runner.Progress).ConfigureAwait(false); if (!exists) { await this.InternalCreateScopeInfoTableAsync(context, DbScopeType.ServerHistory, runner.Connection, runner.Transaction, runner.CancellationToken, runner.Progress).ConfigureAwait(false); } ServerScopeInfo serverScopeInfo; (context, serverScopeInfo) = await this.InternalLoadServerScopeInfoAsync(context, runner.Connection, runner.Transaction, runner.CancellationToken, runner.Progress).ConfigureAwait(false); if (serverScopeInfo == null) { serverScopeInfo = this.InternalCreateScopeInfo(context.ScopeName, DbScopeType.Server) as ServerScopeInfo; (context, serverScopeInfo) = await this.InternalSaveServerScopeInfoAsync(serverScopeInfo, context, runner.Connection, runner.Transaction, runner.CancellationToken, runner.Progress).ConfigureAwait(false); } // Raise error only on server side, since we can't do nothing if we don't have any tables provisionned and no setup provided if ((serverScopeInfo.Setup == null || serverScopeInfo.Schema == null) && (setup == null || setup.Tables.Count <= 0)) { throw new MissingServerScopeTablesException(context.ScopeName); } // if serverscopeinfo is a new, because we never run any sync before, grab schema and affect setup if (setup != null && setup.Tables.Count > 0) { if ((serverScopeInfo.Setup == null && serverScopeInfo.Schema == null) || overwrite) { SyncSet schema; (context, schema) = await this.InternalGetSchemaAsync(context, setup, runner.Connection, runner.Transaction, runner.CancellationToken, runner.Progress).ConfigureAwait(false); serverScopeInfo.Setup = setup; serverScopeInfo.Schema = schema; // Checking if we have already some scopes // Then gets the first scope to get the id List <ServerScopeInfo> allScopes; (context, allScopes) = await this.InternalLoadAllServerScopesInfosAsync(context, runner.Connection, runner.Transaction, runner.CancellationToken, runner.Progress).ConfigureAwait(false); if (allScopes.Count > 0) { // Get the first scope with an existing setup var firstScope = allScopes.FirstOrDefault(sc => sc.Setup != null); if (firstScope != null) { if (serverScopeInfo.Setup.TrackingTablesPrefix != firstScope.Setup.TrackingTablesPrefix) { throw new Exception($"Can't add a new setup with different tracking table prefix. Please use same tracking table prefix as your first setup ([\"{firstScope.Setup.TrackingTablesPrefix}\"])"); } if (serverScopeInfo.Setup.TrackingTablesSuffix != firstScope.Setup.TrackingTablesSuffix) { throw new Exception($"Can't add a new setup with different tracking table suffix. Please use same tracking table suffix as your first setup ([\"{firstScope.Setup.TrackingTablesSuffix}\"])"); } if (serverScopeInfo.Setup.TriggersPrefix != firstScope.Setup.TriggersPrefix) { throw new Exception($"Can't add a new setup with different trigger prefix. Please use same trigger prefix as your first setup ([\"{firstScope.Setup.TriggersPrefix}\"])"); } if (serverScopeInfo.Setup.TriggersSuffix != firstScope.Setup.TriggersSuffix) { throw new Exception($"Can't add a new setup with different trigger suffix. Please use same trigger suffix as your first setup ([\"{firstScope.Setup.TriggersSuffix}\"])"); } } } // Write scopes locally (context, serverScopeInfo) = await this.InternalSaveServerScopeInfoAsync(serverScopeInfo, context, runner.Connection, runner.Transaction, runner.CancellationToken, runner.Progress).ConfigureAwait(false); // override default value that is always false after saving serverScopeInfo.IsNewScope = true; } } await runner.CommitAsync().ConfigureAwait(false); return(context, serverScopeInfo); } catch (Exception ex) { throw GetSyncError(context, ex); } }
/// <summary> /// Be sure all tables are ready and configured for sync /// the ScopeSet Configuration MUST be filled by the schema form Database /// </summary> public virtual async Task <SyncContext> ProvisionAsync(SyncContext context, SyncSet schema, SyncSetup setup, SyncProvision provision, string scopeInfoTableName, DbConnection connection, DbTransaction transaction, CancellationToken cancellationToken, IProgress <ProgressArgs> progress) { if (schema.Tables == null || !schema.HasTables) { throw new MissingTablesException(); } this.Orchestrator.logger.LogDebug(SyncEventsId.Provision, new { TablesCount = schema.Tables.Count, ScopeInfoTableName = scopeInfoTableName }); // get Database builder var builder = this.GetDatabaseBuilder(); builder.UseChangeTracking = this.UseChangeTracking; builder.UseBulkProcedures = this.SupportBulkOperations; // Initialize database if needed await builder.EnsureDatabaseAsync(connection, transaction).ConfigureAwait(false); // Shoudl we create scope if (provision.HasFlag(SyncProvision.ClientScope)) { context = await this.EnsureClientScopeAsync(context, scopeInfoTableName, connection, transaction, cancellationToken, progress).ConfigureAwait(false); } if (provision.HasFlag(SyncProvision.ServerScope)) { context = await this.EnsureServerScopeAsync(context, scopeInfoTableName, connection, transaction, cancellationToken, progress).ConfigureAwait(false); } if (provision.HasFlag(SyncProvision.ServerHistoryScope)) { context = await this.EnsureServerHistoryScopeAsync(context, scopeInfoTableName, connection, transaction, cancellationToken, progress).ConfigureAwait(false); } // Sorting tables based on dependencies between them var schemaTables = schema.Tables .SortByDependencies(tab => tab.GetRelations() .Select(r => r.GetParentTable())); foreach (var schemaTable in schemaTables) { var tableBuilder = this.GetTableBuilder(schemaTable, setup); // set if the builder supports creating the bulk operations proc stock tableBuilder.UseBulkProcedures = this.SupportBulkOperations; tableBuilder.UseChangeTracking = this.UseChangeTracking; // adding filter this.AddFilters(schemaTable, tableBuilder); this.Orchestrator.logger.LogDebug(SyncEventsId.Provision, schemaTable); // Interceptor await this.Orchestrator.InterceptAsync(new TableProvisioningArgs(context, provision, tableBuilder, connection, transaction), cancellationToken).ConfigureAwait(false); await tableBuilder.CreateAsync(provision, connection, transaction).ConfigureAwait(false); await tableBuilder.CreateForeignKeysAsync(connection, transaction).ConfigureAwait(false); // Interceptor await this.Orchestrator.InterceptAsync(new TableProvisionedArgs(context, provision, schemaTable, connection, transaction), cancellationToken).ConfigureAwait(false); } return(context); }
public ServerScopeLoadingArgs(SyncContext context, string scopeName, string scopeTableInfoName, DbConnection connection, DbTransaction transaction) : base(context, connection, transaction) { this.ScopeName = scopeName; this.ScopeTableInfoName = scopeTableInfoName; }
public TrackingTableDroppedArgs(SyncContext context, SyncTable table, ParserName trackingTableName, DbConnection connection = null, DbTransaction transaction = null) : base(context, connection, transaction) { this.Table = table; this.TrackingTableName = trackingTableName; }
public ServerScopeLoadedArgs(SyncContext context, ServerScopeInfo scope, DbConnection connection = null, DbTransaction transaction = null) : base(context, connection, transaction) { this.ScopeInfo = scope; }
public ServerScopeInfoLoadedArgs(SyncContext context, string scopeName, ServerScopeInfo serverScopeInfo, DbConnection connection = null, DbTransaction transaction = null) : base(context, connection, transaction) { this.ScopeName = scopeName; this.ServerScopeInfo = serverScopeInfo; }
/// <summary> /// Update a metadata row /// </summary> internal async Task <(SyncContext context, bool metadataUpdated)> InternalUpdateMetadatasAsync(IScopeInfo scopeInfo, SyncContext context, DbSyncAdapter syncAdapter, SyncRow row, Guid?senderScopeId, bool forceWrite, DbConnection connection, DbTransaction transaction) { context.SyncStage = SyncStage.ChangesApplying; var(command, _) = await syncAdapter.GetCommandAsync(DbCommandType.UpdateMetadata, connection, transaction); if (command == null) { return(context, false); } // Set the parameters value from row syncAdapter.SetColumnParametersValues(command, row); // Set the special parameters for update syncAdapter.AddScopeParametersValues(command, senderScopeId, 0, row.RowState == DataRowState.Deleted, forceWrite); await this.InterceptAsync(new DbCommandArgs(context, command, connection, transaction)).ConfigureAwait(false); var metadataUpdatedRowsCount = await command.ExecuteNonQueryAsync().ConfigureAwait(false); // Check if we have a return value instead var syncRowCountParam = DbSyncAdapter.GetParameter(command, "sync_row_count"); if (syncRowCountParam != null) { metadataUpdatedRowsCount = (int)syncRowCountParam.Value; } command.Dispose(); return(context, metadataUpdatedRowsCount > 0); }
public ScopeTableCreatedArgs(SyncContext context, string scopeName, DbScopeType scopeType, DbConnection connection = null, DbTransaction transaction = null) : base(context, connection, transaction) { this.ScopeType = scopeType; this.ScopeName = scopeName; }
InternalDeleteMetadatasAsync( IEnumerable <IScopeInfo> scopeInfos, SyncContext context, long timestampLimit, DbConnection connection, DbTransaction transaction, CancellationToken cancellationToken, IProgress <ProgressArgs> progress) { context.SyncStage = SyncStage.ChangesApplying; var databaseMetadatasCleaned = new DatabaseMetadatasCleaned { TimestampLimit = timestampLimit }; await this.InterceptAsync(new MetadataCleaningArgs(context, scopeInfos, timestampLimit, connection, transaction), progress, cancellationToken).ConfigureAwait(false); // contains all tables already processed var doneList = new List <SetupTable>(); foreach (var scopeInfo in scopeInfos) { if (scopeInfo.Setup?.Tables == null || scopeInfo.Setup.Tables.Count <= 0) { continue; } foreach (var setupTable in scopeInfo.Setup.Tables) { var isDone = doneList.Any(t => t.EqualsByName(setupTable)); if (isDone) { continue; } // create a fake syncTable // Don't need anything else than table name to make a delete metadata clean up var syncTable = new SyncTable(setupTable.TableName, setupTable.SchemaName); // Create sync adapter var syncAdapter = this.GetSyncAdapter(syncTable, scopeInfo); var(command, _) = await syncAdapter.GetCommandAsync(DbCommandType.DeleteMetadata, connection, transaction); if (command != null) { // Set the special parameters for delete metadata DbSyncAdapter.SetParameterValue(command, "sync_row_timestamp", timestampLimit); await this.InterceptAsync(new DbCommandArgs(context, command, connection, transaction), progress, cancellationToken).ConfigureAwait(false); var rowsCleanedCount = await command.ExecuteNonQueryAsync().ConfigureAwait(false); // Check if we have a return value instead var syncRowCountParam = DbSyncAdapter.GetParameter(command, "sync_row_count"); if (syncRowCountParam != null) { rowsCleanedCount = (int)syncRowCountParam.Value; } // Only add a new table metadata stats object, if we have, at least, purged 1 or more rows if (rowsCleanedCount > 0) { var tableMetadatasCleaned = new TableMetadatasCleaned(syncTable.TableName, syncTable.SchemaName) { RowsCleanedCount = rowsCleanedCount, TimestampLimit = timestampLimit }; databaseMetadatasCleaned.Tables.Add(tableMetadatasCleaned); } command.Dispose(); } doneList.Add(setupTable); } } await this.InterceptAsync(new MetadataCleanedArgs(context, databaseMetadatasCleaned, connection), progress, cancellationToken).ConfigureAwait(false); return(context, databaseMetadatasCleaned); }
/// <summary> /// Constructor /// </summary> public ProgressArgs(SyncContext context, DbConnection connection, DbTransaction transaction) { this.Context = context; this.Connection = connection; this.Transaction = transaction; }
/// <summary> /// Gets a batch of changes to synchronize when given batch size, /// destination knowledge, and change data retriever parameters. /// </summary> /// <returns>A DbSyncContext object that will be used to retrieve the modified data.</returns> public virtual async Task <(SyncContext, BatchInfo, DatabaseChangesSelected)> GetChangeBatchAsync( SyncContext context, MessageGetChangesBatch message, DbConnection connection, DbTransaction transaction, CancellationToken cancellationToken, IProgress <ProgressArgs> progress = null) { // batch info containing changes BatchInfo batchInfo; // Statistics about changes that are selected DatabaseChangesSelected changesSelected; if (context.SyncWay == SyncWay.Upload && context.SyncType == SyncType.Reinitialize) { (batchInfo, changesSelected) = this.GetEmptyChanges(message); return(context, batchInfo, changesSelected); } // Check if the provider is not outdated var isOutdated = this.IsRemoteOutdated(); // Get a chance to make the sync even if it's outdated if (isOutdated) { var outdatedArgs = new OutdatedArgs(context, null, null); // Interceptor await this.InterceptAsync(outdatedArgs).ConfigureAwait(false); if (outdatedArgs.Action != OutdatedAction.Rollback) { context.SyncType = outdatedArgs.Action == OutdatedAction.Reinitialize ? SyncType.Reinitialize : SyncType.ReinitializeWithUpload; } if (outdatedArgs.Action == OutdatedAction.Rollback) { throw new OutOfDateException(); } } // create local directory if (message.BatchSize > 0 && !string.IsNullOrEmpty(message.BatchDirectory) && !Directory.Exists(message.BatchDirectory)) { Directory.CreateDirectory(message.BatchDirectory); } // numbers of batch files generated var batchIndex = 0; // Check if we are in batch mode var isBatch = message.BatchSize > 0; // Create stats object to store changes count var changes = new DatabaseChangesSelected(); // create the in memory changes set var changesSet = new SyncSet(message.Schema.ScopeName); // Create a Schema set without readonly columns, attached to memory changes foreach (var table in message.Schema.Tables) { DbSyncAdapter.CreateChangesTable(message.Schema.Tables[table.TableName, table.SchemaName], changesSet); } // Create a batch info in memory (if !isBatch) or serialized on disk (if isBatch) // batchinfo generate a schema clone with scope columns if needed batchInfo = new BatchInfo(!isBatch, changesSet, message.BatchDirectory); // Clear tables, we will add only the ones we need in the batch info changesSet.Clear(); foreach (var syncTable in message.Schema.Tables) { // if we are in upload stage, so check if table is not download only if (context.SyncWay == SyncWay.Upload && syncTable.SyncDirection == SyncDirection.DownloadOnly) { continue; } // if we are in download stage, so check if table is not download only if (context.SyncWay == SyncWay.Download && syncTable.SyncDirection == SyncDirection.UploadOnly) { continue; } var tableBuilder = this.GetTableBuilder(syncTable); var syncAdapter = tableBuilder.CreateSyncAdapter(connection, transaction); // raise before event context.SyncStage = SyncStage.TableChangesSelecting; var tableChangesSelectingArgs = new TableChangesSelectingArgs(context, syncTable.TableName, connection, transaction); // launch interceptor if any await this.InterceptAsync(tableChangesSelectingArgs).ConfigureAwait(false); // Get Command var selectIncrementalChangesCommand = this.GetSelectChangesCommand(context, syncAdapter, syncTable, message.IsNew); // Set parameters this.SetSelectChangesCommonParameters(context, syncTable, message.ExcludingScopeId, message.IsNew, message.LastTimestamp, selectIncrementalChangesCommand); // Statistics var tableChangesSelected = new TableChangesSelected(syncTable.TableName); // Get the reader using (var dataReader = selectIncrementalChangesCommand.ExecuteReader()) { // memory size total double rowsMemorySize = 0L; // Create a chnages table with scope columns var changesSetTable = DbSyncAdapter.CreateChangesTable(message.Schema.Tables[syncTable.TableName, syncTable.SchemaName], changesSet); while (dataReader.Read()) { // Create a row from dataReader var row = CreateSyncRowFromReader(dataReader, changesSetTable); // Add the row to the changes set changesSetTable.Rows.Add(row); // Set the correct state to be applied if (row.RowState == DataRowState.Deleted) { tableChangesSelected.Deletes++; } else if (row.RowState == DataRowState.Modified) { tableChangesSelected.Upserts++; } // calculate row size if in batch mode if (isBatch) { var fieldsSize = ContainerTable.GetRowSizeFromDataRow(row.ToArray()); var finalFieldSize = fieldsSize / 1024d; if (finalFieldSize > message.BatchSize) { throw new RowOverSizedException(finalFieldSize.ToString()); } // Calculate the new memory size rowsMemorySize += finalFieldSize; // Next line if we don't reach the batch size yet. if (rowsMemorySize <= message.BatchSize) { continue; } // add changes to batchinfo batchInfo.AddChanges(changesSet, batchIndex, false); // increment batch index batchIndex++; // we know the datas are serialized here, so we can flush the set changesSet.Clear(); // Recreate an empty ContainerSet and a ContainerTable changesSet = new SyncSet(message.Schema.ScopeName); changesSetTable = DbSyncAdapter.CreateChangesTable(message.Schema.Tables[syncTable.TableName, syncTable.SchemaName], changesSet); // Init the row memory size rowsMemorySize = 0L; } } } selectIncrementalChangesCommand.Dispose(); context.SyncStage = SyncStage.TableChangesSelected; if (tableChangesSelected.Deletes > 0 || tableChangesSelected.Upserts > 0) { changes.TableChangesSelected.Add(tableChangesSelected); } // Event progress & interceptor context.SyncStage = SyncStage.TableChangesSelected; var tableChangesSelectedArgs = new TableChangesSelectedArgs(context, tableChangesSelected, connection, transaction); this.ReportProgress(context, progress, tableChangesSelectedArgs); await this.InterceptAsync(tableChangesSelectedArgs).ConfigureAwait(false); } // We are in batch mode, and we are at the last batchpart info // Even if we don't have rows inside, we return the changesSet, since it contains at leaset schema if (changesSet != null && changesSet.HasTables) { batchInfo.AddChanges(changesSet, batchIndex, true); } // Check the last index as the last batch batchInfo.EnsureLastBatch(); return(context, batchInfo, changes); }
/// <summary> /// Enumerate all internal changes, no batch mode /// </summary> internal async Task <(BatchInfo, ChangesSelected)> EnumerateChangesInternal( SyncContext context, ScopeInfo scopeInfo, DmSet configTables, string batchDirectory, ConflictResolutionPolicy policy, ICollection <FilterClause> filters) { // create the in memory changes set DmSet changesSet = new DmSet(SyncConfiguration.DMSET_NAME); // Create the batch info, in memory var batchInfo = new BatchInfo(); batchInfo.InMemory = true; using (var connection = this.CreateConnection()) { // Open the connection await connection.OpenAsync(); using (var transaction = connection.BeginTransaction()) { try { // changes that will be returned as selected changes ChangesSelected changes = new ChangesSelected(); foreach (var tableDescription in configTables.Tables) { // if we are in upload stage, so check if table is not download only if (context.SyncWay == SyncWay.Upload && tableDescription.SyncDirection == SyncDirection.DownloadOnly) { continue; } // if we are in download stage, so check if table is not download only if (context.SyncWay == SyncWay.Download && tableDescription.SyncDirection == SyncDirection.UploadOnly) { continue; } var builder = this.GetDatabaseBuilder(tableDescription); var syncAdapter = builder.CreateSyncAdapter(connection, transaction); syncAdapter.ConflictApplyAction = SyncConfiguration.GetApplyAction(policy); // raise before event context.SyncStage = SyncStage.TableChangesSelecting; var beforeArgs = new TableChangesSelectingEventArgs(this.ProviderTypeName, context.SyncStage, tableDescription.TableName); this.TryRaiseProgressEvent(beforeArgs, this.TableChangesSelecting); // selected changes for the current table TableChangesSelected tableSelectedChanges = new TableChangesSelected { TableName = tableDescription.TableName }; // Get Command DbCommand selectIncrementalChangesCommand; DbCommandType dbCommandType; if (this.CanBeServerProvider && context.Parameters != null && context.Parameters.Count > 0 && filters != null && filters.Count > 0) { var filtersName = filters .Where(f => f.TableName.Equals(tableDescription.TableName, StringComparison.InvariantCultureIgnoreCase)) .Select(f => f.ColumnName); if (filtersName != null && filtersName.Count() > 0) { dbCommandType = DbCommandType.SelectChangesWitFilters; selectIncrementalChangesCommand = syncAdapter.GetCommand(dbCommandType, filtersName); } else { dbCommandType = DbCommandType.SelectChanges; selectIncrementalChangesCommand = syncAdapter.GetCommand(dbCommandType); } } else { dbCommandType = DbCommandType.SelectChanges; selectIncrementalChangesCommand = syncAdapter.GetCommand(dbCommandType); } if (selectIncrementalChangesCommand == null) { var exc = "Missing command 'SelectIncrementalChangesCommand' "; throw new Exception(exc); } // Deriving Parameters syncAdapter.SetCommandParameters(dbCommandType, selectIncrementalChangesCommand); // Get a clone of the table with tracking columns var dmTableChanges = BuildChangesTable(tableDescription.TableName, configTables); SetSelectChangesCommonParameters(context, scopeInfo, selectIncrementalChangesCommand); // Set filter parameters if any if (this.CanBeServerProvider && context.Parameters != null && context.Parameters.Count > 0 && filters != null && filters.Count > 0) { var tableFilters = filters.Where(f => f.TableName.Equals(tableDescription.TableName, StringComparison.InvariantCultureIgnoreCase)).ToList(); if (tableFilters != null && tableFilters.Count > 0) { foreach (var filter in tableFilters) { var parameter = context.Parameters.FirstOrDefault(p => p.ColumnName.Equals(filter.ColumnName, StringComparison.InvariantCultureIgnoreCase) && p.TableName.Equals(filter.TableName, StringComparison.InvariantCultureIgnoreCase)); if (parameter != null) { DbManager.SetParameterValue(selectIncrementalChangesCommand, parameter.ColumnName, parameter.Value); } } } } this.AddTrackingColumns <int>(dmTableChanges, "sync_row_is_tombstone"); // Get the reader using (var dataReader = selectIncrementalChangesCommand.ExecuteReader()) { while (dataReader.Read()) { DmRow dataRow = CreateRowFromReader(dataReader, dmTableChanges); //DmRow dataRow = dmTableChanges.NewRow(); // assuming the row is not inserted / modified DmRowState state = DmRowState.Unchanged; // get if the current row is inserted, modified, deleted state = GetStateFromDmRow(dataRow, scopeInfo); if (state != DmRowState.Deleted && state != DmRowState.Modified && state != DmRowState.Added) { continue; } // add row dmTableChanges.Rows.Add(dataRow); // acceptchanges before modifying dataRow.AcceptChanges(); tableSelectedChanges.TotalChanges++; // Set the correct state to be applied if (state == DmRowState.Deleted) { dataRow.Delete(); tableSelectedChanges.Deletes++; } else if (state == DmRowState.Added) { dataRow.SetAdded(); tableSelectedChanges.Inserts++; } else if (state == DmRowState.Modified) { dataRow.SetModified(); tableSelectedChanges.Updates++; } } // Since we dont need this column anymore, remove it this.RemoveTrackingColumns(dmTableChanges, "sync_row_is_tombstone"); // add it to the DmSet changesSet.Tables.Add(dmTableChanges); } // add the stats to global stats changes.TableChangesSelected.Add(tableSelectedChanges); // Raise event for this table context.SyncStage = SyncStage.TableChangesSelected; var args = new TableChangesSelectedEventArgs(this.ProviderTypeName, SyncStage.TableChangesSelected, tableSelectedChanges); this.TryRaiseProgressEvent(args, this.TableChangesSelected); } transaction.Commit(); // generate the batchpartinfo batchInfo.GenerateBatchInfo(0, changesSet, batchDirectory); // Create a new in-memory batch info with an the changes DmSet return(batchInfo, changes); } catch (Exception dbException) { throw; } finally { if (connection != null && connection.State == ConnectionState.Open) { connection.Close(); } } } } }
public SchemaArgs(SyncContext context, SyncSet schema, DbConnection connection, DbTransaction transaction) : base(context, connection, transaction) => this.Schema = schema;