/// <summary> /// Generate an empty BatchInfo /// </summary> internal (BatchInfo, ChangesSelected) GetEmptyChanges(SyncContext context, ScopeInfo scopeInfo, int downloadBatchSizeInKB, string batchDirectory) { // Get config var isBatched = downloadBatchSizeInKB > 0; // create the in memory changes set DmSet changesSet = new DmSet(SyncConfiguration.DMSET_NAME); // Create the batch info, in memory var batchInfo = new BatchInfo { InMemory = !isBatched }; if (isBatched) { batchInfo.Directory = BatchInfo.GenerateNewDirectoryName(); } // generate the batchpartinfo var bpi = batchInfo.GenerateBatchInfo(0, changesSet, batchDirectory); bpi.IsLastBatch = true; // Create a new in-memory batch info with an the changes DmSet return(batchInfo, new ChangesSelected()); }
/// <summary> /// Generate an empty BatchInfo /// </summary> internal (BatchInfo, ChangesSelected) GetEmptyChanges(SyncContext context, ScopeInfo scopeInfo) { // Get config var configuration = GetCacheConfiguration(); var isBatched = configuration.DownloadBatchSizeInKB > 0; // create the in memory changes set DmSet changesSet = new DmSet(configuration.ScopeSet.DmSetName); // Create the batch info, in memory var batchInfo = new BatchInfo(); batchInfo.InMemory = !isBatched; if (!isBatched) { batchInfo.Directory = BatchInfo.GenerateNewDirectoryName(); } // generate the batchpartinfo var bpi = batchInfo.GenerateBatchInfo(0, changesSet, configuration.BatchDirectory); bpi.IsLastBatch = true; // Create a new in-memory batch info with an the changes DmSet return(batchInfo, new ChangesSelected()); }
/// <summary> /// Enumerate all internal changes, no batch mode /// </summary> internal async Task <(BatchInfo, ChangesSelected)> EnumerateChangesInBatchesInternal (SyncContext context, ScopeInfo scopeInfo, int downloadBatchSizeInKB, DmSet configTables, string batchDirectory, ConflictResolutionPolicy policy, ICollection <FilterClause> filters) { DmTable dmTable = null; // memory size total double memorySizeFromDmRows = 0L; int batchIndex = 0; // this batch info won't be in memory, it will be be batched BatchInfo batchInfo = new BatchInfo { // directory where all files will be stored Directory = BatchInfo.GenerateNewDirectoryName(), // not in memory since we serialized all files in the tmp directory InMemory = false }; // Create stats object to store changes count ChangesSelected changes = new ChangesSelected(); using (var connection = this.CreateConnection()) { try { // Open the connection await connection.OpenAsync(); using (var transaction = connection.BeginTransaction()) { // create the in memory changes set DmSet changesSet = new DmSet(configTables.DmSetName); foreach (var tableDescription in configTables.Tables) { // if we are in upload stage, so check if table is not download only if (context.SyncWay == SyncWay.Upload && tableDescription.SyncDirection == SyncDirection.DownloadOnly) { continue; } // if we are in download stage, so check if table is not download only if (context.SyncWay == SyncWay.Download && tableDescription.SyncDirection == SyncDirection.UploadOnly) { continue; } var builder = this.GetDatabaseBuilder(tableDescription); var syncAdapter = builder.CreateSyncAdapter(connection, transaction); syncAdapter.ConflictApplyAction = SyncConfiguration.GetApplyAction(policy); // raise before event context.SyncStage = SyncStage.TableChangesSelecting; var beforeArgs = new TableChangesSelectingEventArgs(this.ProviderTypeName, context.SyncStage, tableDescription.TableName); this.TryRaiseProgressEvent(beforeArgs, this.TableChangesSelecting); // Get Command DbCommand selectIncrementalChangesCommand; DbCommandType dbCommandType; if (this.CanBeServerProvider && context.Parameters != null && context.Parameters.Count > 0 && filters != null && filters.Count > 0) { var filtersName = filters .Where(f => f.TableName.Equals(tableDescription.TableName, StringComparison.InvariantCultureIgnoreCase)) .Select(f => f.ColumnName); if (filtersName != null && filtersName.Count() > 0) { dbCommandType = DbCommandType.SelectChangesWitFilters; selectIncrementalChangesCommand = syncAdapter.GetCommand(dbCommandType, filtersName); } else { dbCommandType = DbCommandType.SelectChanges; selectIncrementalChangesCommand = syncAdapter.GetCommand(dbCommandType); } } else { dbCommandType = DbCommandType.SelectChanges; selectIncrementalChangesCommand = syncAdapter.GetCommand(dbCommandType); } // Deriving Parameters syncAdapter.SetCommandParameters(DbCommandType.SelectChanges, selectIncrementalChangesCommand); if (selectIncrementalChangesCommand == null) { var exc = "Missing command 'SelectIncrementalChangesCommand' "; throw new Exception(exc); } dmTable = BuildChangesTable(tableDescription.TableName, configTables); try { // Set commons parameters SetSelectChangesCommonParameters(context, scopeInfo, selectIncrementalChangesCommand); // Set filter parameters if any // Only on server side if (this.CanBeServerProvider && context.Parameters != null && context.Parameters.Count > 0 && filters != null && filters.Count > 0) { var filterTable = filters.Where(f => f.TableName.Equals(tableDescription.TableName, StringComparison.InvariantCultureIgnoreCase)).ToList(); if (filterTable != null && filterTable.Count > 0) { foreach (var filter in filterTable) { var parameter = context.Parameters.FirstOrDefault(p => p.ColumnName.Equals(filter.ColumnName, StringComparison.InvariantCultureIgnoreCase) && p.TableName.Equals(filter.TableName, StringComparison.InvariantCultureIgnoreCase)); if (parameter != null) { DbManager.SetParameterValue(selectIncrementalChangesCommand, parameter.ColumnName, parameter.Value); } } } } this.AddTrackingColumns <int>(dmTable, "sync_row_is_tombstone"); // Statistics TableChangesSelected tableChangesSelected = new TableChangesSelected { TableName = tableDescription.TableName }; changes.TableChangesSelected.Add(tableChangesSelected); // Get the reader using (var dataReader = selectIncrementalChangesCommand.ExecuteReader()) { while (dataReader.Read()) { DmRow dmRow = CreateRowFromReader(dataReader, dmTable); DmRowState state = DmRowState.Unchanged; state = GetStateFromDmRow(dmRow, scopeInfo); // If the row is not deleted inserted or modified, go next if (state != DmRowState.Deleted && state != DmRowState.Modified && state != DmRowState.Added) { continue; } var fieldsSize = DmTableSurrogate.GetRowSizeFromDataRow(dmRow); var dmRowSize = fieldsSize / 1024d; if (dmRowSize > downloadBatchSizeInKB) { var exc = $"Row is too big ({dmRowSize} kb.) for the current Configuration.DownloadBatchSizeInKB ({downloadBatchSizeInKB} kb.) Aborting Sync..."; throw new Exception(exc); } // Calculate the new memory size memorySizeFromDmRows = memorySizeFromDmRows + dmRowSize; // add row dmTable.Rows.Add(dmRow); tableChangesSelected.TotalChanges++; // acceptchanges before modifying dmRow.AcceptChanges(); // Set the correct state to be applied if (state == DmRowState.Deleted) { dmRow.Delete(); tableChangesSelected.Deletes++; } else if (state == DmRowState.Added) { dmRow.SetAdded(); tableChangesSelected.Inserts++; } else if (state == DmRowState.Modified) { dmRow.SetModified(); tableChangesSelected.Updates++; } // We exceed the memorySize, so we can add it to a batch if (memorySizeFromDmRows > downloadBatchSizeInKB) { // Since we dont need this column anymore, remove it this.RemoveTrackingColumns(dmTable, "sync_row_is_tombstone"); changesSet.Tables.Add(dmTable); // generate the batch part info batchInfo.GenerateBatchInfo(batchIndex, changesSet, batchDirectory); // increment batch index batchIndex++; changesSet.Clear(); // Recreate an empty DmSet, then a dmTable clone changesSet = new DmSet(configTables.DmSetName); dmTable = dmTable.Clone(); this.AddTrackingColumns <int>(dmTable, "sync_row_is_tombstone"); // Init the row memory size memorySizeFromDmRows = 0L; // add stats for a SyncProgress event context.SyncStage = SyncStage.TableChangesSelected; var args2 = new TableChangesSelectedEventArgs (this.ProviderTypeName, SyncStage.TableChangesSelected, tableChangesSelected); this.TryRaiseProgressEvent(args2, this.TableChangesSelected); } } // Since we dont need this column anymore, remove it this.RemoveTrackingColumns(dmTable, "sync_row_is_tombstone"); context.SyncStage = SyncStage.TableChangesSelected; changesSet.Tables.Add(dmTable); // Init the row memory size memorySizeFromDmRows = 0L; // Event progress context.SyncStage = SyncStage.TableChangesSelected; var args = new TableChangesSelectedEventArgs(this.ProviderTypeName, SyncStage.TableChangesSelected, tableChangesSelected); this.TryRaiseProgressEvent(args, this.TableChangesSelected); } } catch (Exception) { throw; } finally { } } // We are in batch mode, and we are at the last batchpart info if (changesSet != null && changesSet.HasTables && changesSet.HasChanges()) { var batchPartInfo = batchInfo.GenerateBatchInfo(batchIndex, changesSet, batchDirectory); if (batchPartInfo != null) { batchPartInfo.IsLastBatch = true; } } transaction.Commit(); } } catch (Exception) { throw; } finally { if (connection != null && connection.State == ConnectionState.Open) { connection.Close(); } } } return(batchInfo, changes); }
public async Task <(SyncContext, BatchInfo, ChangesStatistics)> GetChangeBatchAsync(SyncContext context, ScopeInfo scopeInfo) { // While we have an other batch to process var isLastBatch = false; // Create the BatchInfo and SyncContext to return at the end BatchInfo changes = new BatchInfo(); changes.Directory = BatchInfo.GenerateNewDirectoryName(); SyncContext syncContext = null; ChangesStatistics changesStatistics = null; while (!isLastBatch) { HttpMessage httpMessage = new HttpMessage(); httpMessage.SyncContext = context; httpMessage.Step = HttpStep.GetChangeBatch; httpMessage.GetChangeBatch = new HttpGetChangeBatchMessage { ScopeInfo = scopeInfo, BatchIndexRequested = changes.BatchIndex }; var httpMessageResponse = await this.httpRequestHandler.ProcessRequest(context, httpMessage, cancellationToken); if (httpMessageResponse == null) { throw new Exception("Can't have an empty body"); } if (httpMessageResponse.GetChangeBatch == null) { throw new Exception("Can't have an empty GetChangeBatch"); } changesStatistics = httpMessageResponse.GetChangeBatch.ChangesStatistics; changes.InMemory = httpMessageResponse.GetChangeBatch.InMemory; syncContext = httpMessageResponse.SyncContext; // get the bpi and add it to the BatchInfo var bpi = httpMessageResponse.GetChangeBatch.BatchPartInfo; if (bpi != null) { changes.BatchIndex = bpi.Index; changes.BatchPartsInfo.Add(bpi); isLastBatch = bpi.IsLastBatch; } else { changes.BatchIndex = 0; isLastBatch = true; // break the while { } story break; } if (changes.InMemory) { // load the DmSet in memory bpi.Set = httpMessageResponse.GetChangeBatch.Set.ConvertToDmSet(); } else { // Serialize the file ! var bpId = BatchInfo.GenerateNewFileName(changes.BatchIndex.ToString()); var fileName = Path.Combine(this.serviceConfiguration.BatchDirectory, changes.Directory, bpId); BatchPart.Serialize(httpMessageResponse.GetChangeBatch.Set, fileName); bpi.FileName = fileName; } // Clear the DmSetSurrogate from response, we don't need it anymore httpMessageResponse.GetChangeBatch.Set.Dispose(); httpMessageResponse.GetChangeBatch.Set = null; // if not last, increment batchIndex for next request if (!isLastBatch) { changes.BatchIndex++; } } return(syncContext, changes, changesStatistics); }
private async Task <HttpMessage> ApplyChangesAsync(HttpMessage httpMessage) { HttpMessageApplyChanges httpMessageContent; if (httpMessage.Content is HttpMessageApplyChanges) { httpMessageContent = httpMessage.Content as HttpMessageApplyChanges; } else { httpMessageContent = (httpMessage.Content as JObject).ToObject <HttpMessageApplyChanges>(); } if (httpMessageContent == null) { throw new ArgumentException("ApplyChanges message could not be null"); } var scopeInfo = httpMessageContent.FromScope; if (scopeInfo == null) { throw new ArgumentException("ApplyChanges ScopeInfo could not be null"); } var schema = httpMessageContent.Schema.ConvertToDmSet(); BatchInfo batchInfo; var bpi = httpMessageContent.BatchPartInfo; if (httpMessageContent.InMemory) { batchInfo = new BatchInfo { BatchIndex = 0, BatchPartsInfo = new List <BatchPartInfo>(new[] { bpi }), InMemory = true }; bpi.Set = httpMessageContent.Set.ConvertToDmSet(); httpMessageContent.Set.Dispose(); httpMessageContent.Set = null; var(c, s) = await this.ApplyChangesAsync(httpMessage.SyncContext, new MessageApplyChanges { FromScope = scopeInfo, Schema = schema, Policy = httpMessageContent.Policy, UseBulkOperations = httpMessageContent.UseBulkOperations, ScopeInfoTableName = httpMessageContent.ScopeInfoTableName, Changes = batchInfo }); httpMessageContent.ChangesApplied = s; httpMessageContent.BatchPartInfo.Clear(); httpMessageContent.BatchPartInfo.FileName = null; httpMessage.SyncContext = c; httpMessage.Content = httpMessageContent; return(httpMessage); } // not in memory batchInfo = this.LocalProvider.CacheManager.GetValue <BatchInfo>("ApplyChanges_BatchInfo"); if (batchInfo == null) { batchInfo = new BatchInfo { BatchIndex = 0, BatchPartsInfo = new List <BatchPartInfo>(new[] { bpi }), InMemory = false, Directory = BatchInfo.GenerateNewDirectoryName() }; } else { batchInfo.BatchPartsInfo.Add(bpi); } var bpId = BatchInfo.GenerateNewFileName(httpMessageContent.BatchIndex.ToString()); // to save the file, we should use the local configuration batch directory var fileName = Path.Combine(this.Configuration.BatchDirectory, batchInfo.Directory, bpId); BatchPart.Serialize(httpMessageContent.Set, fileName); bpi.FileName = fileName; this.LocalProvider.CacheManager.Set("ApplyChanges_BatchInfo", batchInfo); // Clear the httpMessage set if (httpMessageContent != null) { httpMessageContent.Set.Dispose(); httpMessageContent.Set = null; } // if it's last batch sent if (bpi.IsLastBatch) { var(c, s) = await this.ApplyChangesAsync(httpMessage.SyncContext, new MessageApplyChanges { FromScope = scopeInfo, Schema = schema, Policy = httpMessageContent.Policy, UseBulkOperations = httpMessageContent.UseBulkOperations, ScopeInfoTableName = httpMessageContent.ScopeInfoTableName, Changes = batchInfo }); this.LocalProvider.CacheManager.Remove("ApplyChanges_BatchInfo"); httpMessage.SyncContext = c; httpMessageContent.ChangesApplied = s; } httpMessageContent.BatchPartInfo.Clear(); httpMessageContent.BatchPartInfo.FileName = null; httpMessage.Content = httpMessageContent; return(httpMessage); }
public async Task <(SyncContext, BatchInfo, ChangesSelected)> GetChangeBatchAsync( SyncContext context, MessageGetChangesBatch message) { // While we have an other batch to process var isLastBatch = false; // Create the BatchInfo and SyncContext to return at the end BatchInfo changes = new BatchInfo { Directory = BatchInfo.GenerateNewDirectoryName() }; SyncContext syncContext = null; ChangesSelected changesSelected = null; while (!isLastBatch) { HttpMessage httpMessage = new HttpMessage { SyncContext = context, Step = HttpStep.GetChangeBatch, Content = new HttpMessageGetChangesBatch { ScopeInfo = message.ScopeInfo, BatchIndexRequested = changes.BatchIndex, DownloadBatchSizeInKB = message.DownloadBatchSizeInKB, BatchDirectory = message.BatchDirectory, Schema = new DmSetSurrogate(message.Schema), Filters = message.Filters, Policy = message.Policy, SerializationFormat = message.SerializationFormat } }; var httpMessageResponse = await this.httpRequestHandler.ProcessRequest(httpMessage, message.SerializationFormat, cancellationToken); if (httpMessageResponse == null) { throw new Exception("Can't have an empty body"); } HttpMessageGetChangesBatch httpMessageContent; if (httpMessageResponse.Content is HttpMessageGetChangesBatch) { httpMessageContent = httpMessageResponse.Content as HttpMessageGetChangesBatch; } else { httpMessageContent = (httpMessageResponse.Content as JObject).ToObject <HttpMessageGetChangesBatch>(); } if (httpMessageContent == null) { throw new Exception("Can't have an empty GetChangeBatch"); } changesSelected = httpMessageContent.ChangesSelected; changes.InMemory = httpMessageContent.InMemory; syncContext = httpMessageResponse.SyncContext; // get the bpi and add it to the BatchInfo var bpi = httpMessageContent.BatchPartInfo; if (bpi != null) { changes.BatchIndex = bpi.Index; changes.BatchPartsInfo.Add(bpi); isLastBatch = bpi.IsLastBatch; } else { changes.BatchIndex = 0; isLastBatch = true; // break the while { } story break; } if (changes.InMemory) { // load the DmSet in memory bpi.Set = httpMessageContent.Set.ConvertToDmSet(); } else { // Serialize the file ! var bpId = BatchInfo.GenerateNewFileName(changes.BatchIndex.ToString()); var fileName = Path.Combine(message.BatchDirectory, changes.Directory, bpId); BatchPart.Serialize(httpMessageContent.Set, fileName); bpi.FileName = fileName; bpi.Clear(); } // Clear the DmSetSurrogate from response, we don't need it anymore if (httpMessageContent.Set != null) { httpMessageContent.Set.Dispose(); httpMessageContent.Set = null; } // if not last, increment batchIndex for next request if (!isLastBatch) { changes.BatchIndex++; } } return(syncContext, changes, changesSelected); }
private async Task <HttpMessage> ApplyChangesAsync(HttpMessage httpMessage) { if (httpMessage.ApplyChanges == null) { throw new ArgumentException("ApplyChanges message could not be null"); } var scopeInfo = httpMessage.ApplyChanges.ScopeInfo; if (scopeInfo == null) { throw new ArgumentException("ApplyChanges ScopeInfo could not be null"); } BatchInfo batchInfo; var bpi = httpMessage.ApplyChanges.BatchPartInfo; if (httpMessage.ApplyChanges.InMemory) { batchInfo = new BatchInfo { BatchIndex = 0, BatchPartsInfo = new List <BatchPartInfo>(new[] { bpi }), InMemory = true }; bpi.Set = httpMessage.ApplyChanges.Set.ConvertToDmSet(); httpMessage.ApplyChanges.Set.Dispose(); httpMessage.ApplyChanges.Set = null; var(c, s) = await this.ApplyChangesAsync(httpMessage.SyncContext, scopeInfo, batchInfo); httpMessage.SyncContext = c; httpMessage.ApplyChanges.ChangesStatistics = s; httpMessage.ApplyChanges.BatchPartInfo.Clear(); httpMessage.ApplyChanges.BatchPartInfo.FileName = null; return(httpMessage); } // not in memory batchInfo = this.LocalProvider.CacheManager.GetValue <BatchInfo>("ApplyChanges_BatchInfo"); if (batchInfo == null) { batchInfo = new BatchInfo { BatchIndex = 0, BatchPartsInfo = new List <BatchPartInfo>(new[] { bpi }), InMemory = false, Directory = BatchInfo.GenerateNewDirectoryName() }; } else { batchInfo.BatchPartsInfo.Add(bpi); } var bpId = BatchInfo.GenerateNewFileName(httpMessage.ApplyChanges.BatchIndex.ToString()); var fileName = Path.Combine(this.LocalProvider.GetCacheConfiguration().BatchDirectory, batchInfo.Directory, bpId); BatchPart.Serialize(httpMessage.ApplyChanges.Set, fileName); bpi.FileName = fileName; this.LocalProvider.CacheManager.Set("ApplyChanges_BatchInfo", batchInfo); // Clear the httpMessage set if (httpMessage.ApplyChanges != null) { httpMessage.ApplyChanges.Set.Dispose(); httpMessage.ApplyChanges.Set = null; } // if it's last batch sent if (bpi.IsLastBatch) { var(c, s) = await this.ApplyChangesAsync(httpMessage.SyncContext, scopeInfo, batchInfo); this.LocalProvider.CacheManager.Remove("ApplyChanges_BatchInfo"); httpMessage.SyncContext = c; httpMessage.ApplyChanges.ChangesStatistics = s; } httpMessage.ApplyChanges.BatchPartInfo.Clear(); httpMessage.ApplyChanges.BatchPartInfo.FileName = null; return(httpMessage); }