/// <summary>
/// Generate an empty BatchInfo
/// </summary>
internal (BatchInfo, ChangesSelected) GetEmptyChanges(SyncContext context, ScopeInfo scopeInfo,
int downloadBatchSizeInKB, string batchDirectory)
{
// Get config
var isBatched = downloadBatchSizeInKB > 0;
// create the in memory changes set
DmSet changesSet = new DmSet(SyncConfiguration.DMSET_NAME);
// Create the batch info, in memory
var batchInfo = new BatchInfo
{
InMemory = !isBatched
};
if (isBatched)
{
batchInfo.Directory = BatchInfo.GenerateNewDirectoryName();
}
// generate the batchpartinfo
var bpi = batchInfo.GenerateBatchInfo(0, changesSet, batchDirectory);
bpi.IsLastBatch = true;
// Create a new in-memory batch info with an the changes DmSet
return(batchInfo, new ChangesSelected());
}
/// <summary>
/// Generate an empty BatchInfo
/// </summary>
internal (BatchInfo, ChangesSelected) GetEmptyChanges(SyncContext context, ScopeInfo scopeInfo)
{
// Get config
var configuration = GetCacheConfiguration();
var isBatched = configuration.DownloadBatchSizeInKB > 0;
// create the in memory changes set
DmSet changesSet = new DmSet(configuration.ScopeSet.DmSetName);
// Create the batch info, in memory
var batchInfo = new BatchInfo();
batchInfo.InMemory = !isBatched;
if (!isBatched)
{
batchInfo.Directory = BatchInfo.GenerateNewDirectoryName();
}
// generate the batchpartinfo
var bpi = batchInfo.GenerateBatchInfo(0, changesSet, configuration.BatchDirectory);
bpi.IsLastBatch = true;
// Create a new in-memory batch info with an the changes DmSet
return(batchInfo, new ChangesSelected());
}
/// <summary>
/// Enumerate all internal changes, no batch mode
/// </summary>
internal async Task <(BatchInfo, ChangesSelected)> EnumerateChangesInBatchesInternal
(SyncContext context, ScopeInfo scopeInfo, int downloadBatchSizeInKB, DmSet configTables, string batchDirectory, ConflictResolutionPolicy policy, ICollection <FilterClause> filters)
{
DmTable dmTable = null;
// memory size total
double memorySizeFromDmRows = 0L;
int batchIndex = 0;
// this batch info won't be in memory, it will be be batched
BatchInfo batchInfo = new BatchInfo
{
// directory where all files will be stored
Directory = BatchInfo.GenerateNewDirectoryName(),
// not in memory since we serialized all files in the tmp directory
InMemory = false
};
// Create stats object to store changes count
ChangesSelected changes = new ChangesSelected();
using (var connection = this.CreateConnection())
{
try
{
// Open the connection
await connection.OpenAsync();
using (var transaction = connection.BeginTransaction())
{
// create the in memory changes set
DmSet changesSet = new DmSet(configTables.DmSetName);
foreach (var tableDescription in configTables.Tables)
{
// if we are in upload stage, so check if table is not download only
if (context.SyncWay == SyncWay.Upload && tableDescription.SyncDirection == SyncDirection.DownloadOnly)
{
continue;
}
// if we are in download stage, so check if table is not download only
if (context.SyncWay == SyncWay.Download && tableDescription.SyncDirection == SyncDirection.UploadOnly)
{
continue;
}
var builder = this.GetDatabaseBuilder(tableDescription);
var syncAdapter = builder.CreateSyncAdapter(connection, transaction);
syncAdapter.ConflictApplyAction = SyncConfiguration.GetApplyAction(policy);
// raise before event
context.SyncStage = SyncStage.TableChangesSelecting;
var beforeArgs = new TableChangesSelectingEventArgs(this.ProviderTypeName, context.SyncStage, tableDescription.TableName);
this.TryRaiseProgressEvent(beforeArgs, this.TableChangesSelecting);
// Get Command
DbCommand selectIncrementalChangesCommand;
DbCommandType dbCommandType;
if (this.CanBeServerProvider && context.Parameters != null && context.Parameters.Count > 0 && filters != null && filters.Count > 0)
{
var filtersName = filters
.Where(f => f.TableName.Equals(tableDescription.TableName, StringComparison.InvariantCultureIgnoreCase))
.Select(f => f.ColumnName);
if (filtersName != null && filtersName.Count() > 0)
{
dbCommandType = DbCommandType.SelectChangesWitFilters;
selectIncrementalChangesCommand = syncAdapter.GetCommand(dbCommandType, filtersName);
}
else
{
dbCommandType = DbCommandType.SelectChanges;
selectIncrementalChangesCommand = syncAdapter.GetCommand(dbCommandType);
}
}
else
{
dbCommandType = DbCommandType.SelectChanges;
selectIncrementalChangesCommand = syncAdapter.GetCommand(dbCommandType);
}
// Deriving Parameters
syncAdapter.SetCommandParameters(DbCommandType.SelectChanges, selectIncrementalChangesCommand);
if (selectIncrementalChangesCommand == null)
{
var exc = "Missing command 'SelectIncrementalChangesCommand' ";
throw new Exception(exc);
}
dmTable = BuildChangesTable(tableDescription.TableName, configTables);
try
{
// Set commons parameters
SetSelectChangesCommonParameters(context, scopeInfo, selectIncrementalChangesCommand);
// Set filter parameters if any
// Only on server side
if (this.CanBeServerProvider && context.Parameters != null && context.Parameters.Count > 0 && filters != null && filters.Count > 0)
{
var filterTable = filters.Where(f => f.TableName.Equals(tableDescription.TableName, StringComparison.InvariantCultureIgnoreCase)).ToList();
if (filterTable != null && filterTable.Count > 0)
{
foreach (var filter in filterTable)
{
var parameter = context.Parameters.FirstOrDefault(p => p.ColumnName.Equals(filter.ColumnName, StringComparison.InvariantCultureIgnoreCase) && p.TableName.Equals(filter.TableName, StringComparison.InvariantCultureIgnoreCase));
if (parameter != null)
{
DbManager.SetParameterValue(selectIncrementalChangesCommand, parameter.ColumnName, parameter.Value);
}
}
}
}
this.AddTrackingColumns <int>(dmTable, "sync_row_is_tombstone");
// Statistics
TableChangesSelected tableChangesSelected = new TableChangesSelected
{
TableName = tableDescription.TableName
};
changes.TableChangesSelected.Add(tableChangesSelected);
// Get the reader
using (var dataReader = selectIncrementalChangesCommand.ExecuteReader())
{
while (dataReader.Read())
{
DmRow dmRow = CreateRowFromReader(dataReader, dmTable);
DmRowState state = DmRowState.Unchanged;
state = GetStateFromDmRow(dmRow, scopeInfo);
// If the row is not deleted inserted or modified, go next
if (state != DmRowState.Deleted && state != DmRowState.Modified && state != DmRowState.Added)
{
continue;
}
var fieldsSize = DmTableSurrogate.GetRowSizeFromDataRow(dmRow);
var dmRowSize = fieldsSize / 1024d;
if (dmRowSize > downloadBatchSizeInKB)
{
var exc = $"Row is too big ({dmRowSize} kb.) for the current Configuration.DownloadBatchSizeInKB ({downloadBatchSizeInKB} kb.) Aborting Sync...";
throw new Exception(exc);
}
// Calculate the new memory size
memorySizeFromDmRows = memorySizeFromDmRows + dmRowSize;
// add row
dmTable.Rows.Add(dmRow);
tableChangesSelected.TotalChanges++;
// acceptchanges before modifying
dmRow.AcceptChanges();
// Set the correct state to be applied
if (state == DmRowState.Deleted)
{
dmRow.Delete();
tableChangesSelected.Deletes++;
}
else if (state == DmRowState.Added)
{
dmRow.SetAdded();
tableChangesSelected.Inserts++;
}
else if (state == DmRowState.Modified)
{
dmRow.SetModified();
tableChangesSelected.Updates++;
}
// We exceed the memorySize, so we can add it to a batch
if (memorySizeFromDmRows > downloadBatchSizeInKB)
{
// Since we dont need this column anymore, remove it
this.RemoveTrackingColumns(dmTable, "sync_row_is_tombstone");
changesSet.Tables.Add(dmTable);
// generate the batch part info
batchInfo.GenerateBatchInfo(batchIndex, changesSet, batchDirectory);
// increment batch index
batchIndex++;
changesSet.Clear();
// Recreate an empty DmSet, then a dmTable clone
changesSet = new DmSet(configTables.DmSetName);
dmTable = dmTable.Clone();
this.AddTrackingColumns <int>(dmTable, "sync_row_is_tombstone");
// Init the row memory size
memorySizeFromDmRows = 0L;
// add stats for a SyncProgress event
context.SyncStage = SyncStage.TableChangesSelected;
var args2 = new TableChangesSelectedEventArgs
(this.ProviderTypeName, SyncStage.TableChangesSelected, tableChangesSelected);
this.TryRaiseProgressEvent(args2, this.TableChangesSelected);
}
}
// Since we dont need this column anymore, remove it
this.RemoveTrackingColumns(dmTable, "sync_row_is_tombstone");
context.SyncStage = SyncStage.TableChangesSelected;
changesSet.Tables.Add(dmTable);
// Init the row memory size
memorySizeFromDmRows = 0L;
// Event progress
context.SyncStage = SyncStage.TableChangesSelected;
var args = new TableChangesSelectedEventArgs(this.ProviderTypeName, SyncStage.TableChangesSelected, tableChangesSelected);
this.TryRaiseProgressEvent(args, this.TableChangesSelected);
}
}
catch (Exception)
{
throw;
}
finally
{
}
}
// We are in batch mode, and we are at the last batchpart info
if (changesSet != null && changesSet.HasTables && changesSet.HasChanges())
{
var batchPartInfo = batchInfo.GenerateBatchInfo(batchIndex, changesSet, batchDirectory);
if (batchPartInfo != null)
{
batchPartInfo.IsLastBatch = true;
}
}
transaction.Commit();
}
}
catch (Exception)
{
throw;
}
finally
{
if (connection != null && connection.State == ConnectionState.Open)
{
connection.Close();
}
}
}
return(batchInfo, changes);
}
public async Task <(SyncContext, BatchInfo, ChangesStatistics)> GetChangeBatchAsync(SyncContext context, ScopeInfo scopeInfo)
{
// While we have an other batch to process
var isLastBatch = false;
// Create the BatchInfo and SyncContext to return at the end
BatchInfo changes = new BatchInfo();
changes.Directory = BatchInfo.GenerateNewDirectoryName();
SyncContext syncContext = null;
ChangesStatistics changesStatistics = null;
while (!isLastBatch)
{
HttpMessage httpMessage = new HttpMessage();
httpMessage.SyncContext = context;
httpMessage.Step = HttpStep.GetChangeBatch;
httpMessage.GetChangeBatch = new HttpGetChangeBatchMessage
{
ScopeInfo = scopeInfo,
BatchIndexRequested = changes.BatchIndex
};
var httpMessageResponse = await this.httpRequestHandler.ProcessRequest(context, httpMessage, cancellationToken);
if (httpMessageResponse == null)
{
throw new Exception("Can't have an empty body");
}
if (httpMessageResponse.GetChangeBatch == null)
{
throw new Exception("Can't have an empty GetChangeBatch");
}
changesStatistics = httpMessageResponse.GetChangeBatch.ChangesStatistics;
changes.InMemory = httpMessageResponse.GetChangeBatch.InMemory;
syncContext = httpMessageResponse.SyncContext;
// get the bpi and add it to the BatchInfo
var bpi = httpMessageResponse.GetChangeBatch.BatchPartInfo;
if (bpi != null)
{
changes.BatchIndex = bpi.Index;
changes.BatchPartsInfo.Add(bpi);
isLastBatch = bpi.IsLastBatch;
}
else
{
changes.BatchIndex = 0;
isLastBatch = true;
// break the while { } story
break;
}
if (changes.InMemory)
{
// load the DmSet in memory
bpi.Set = httpMessageResponse.GetChangeBatch.Set.ConvertToDmSet();
}
else
{
// Serialize the file !
var bpId = BatchInfo.GenerateNewFileName(changes.BatchIndex.ToString());
var fileName = Path.Combine(this.serviceConfiguration.BatchDirectory, changes.Directory, bpId);
BatchPart.Serialize(httpMessageResponse.GetChangeBatch.Set, fileName);
bpi.FileName = fileName;
}
// Clear the DmSetSurrogate from response, we don't need it anymore
httpMessageResponse.GetChangeBatch.Set.Dispose();
httpMessageResponse.GetChangeBatch.Set = null;
// if not last, increment batchIndex for next request
if (!isLastBatch)
{
changes.BatchIndex++;
}
}
return(syncContext, changes, changesStatistics);
}
private async Task <HttpMessage> ApplyChangesAsync(HttpMessage httpMessage)
{
HttpMessageApplyChanges httpMessageContent;
if (httpMessage.Content is HttpMessageApplyChanges)
{
httpMessageContent = httpMessage.Content as HttpMessageApplyChanges;
}
else
{
httpMessageContent = (httpMessage.Content as JObject).ToObject <HttpMessageApplyChanges>();
}
if (httpMessageContent == null)
{
throw new ArgumentException("ApplyChanges message could not be null");
}
var scopeInfo = httpMessageContent.FromScope;
if (scopeInfo == null)
{
throw new ArgumentException("ApplyChanges ScopeInfo could not be null");
}
var schema = httpMessageContent.Schema.ConvertToDmSet();
BatchInfo batchInfo;
var bpi = httpMessageContent.BatchPartInfo;
if (httpMessageContent.InMemory)
{
batchInfo = new BatchInfo
{
BatchIndex = 0,
BatchPartsInfo = new List <BatchPartInfo>(new[] { bpi }),
InMemory = true
};
bpi.Set = httpMessageContent.Set.ConvertToDmSet();
httpMessageContent.Set.Dispose();
httpMessageContent.Set = null;
var(c, s) = await this.ApplyChangesAsync(httpMessage.SyncContext,
new MessageApplyChanges
{
FromScope = scopeInfo,
Schema = schema,
Policy = httpMessageContent.Policy,
UseBulkOperations = httpMessageContent.UseBulkOperations,
ScopeInfoTableName = httpMessageContent.ScopeInfoTableName,
Changes = batchInfo
});
httpMessageContent.ChangesApplied = s;
httpMessageContent.BatchPartInfo.Clear();
httpMessageContent.BatchPartInfo.FileName = null;
httpMessage.SyncContext = c;
httpMessage.Content = httpMessageContent;
return(httpMessage);
}
// not in memory
batchInfo = this.LocalProvider.CacheManager.GetValue <BatchInfo>("ApplyChanges_BatchInfo");
if (batchInfo == null)
{
batchInfo = new BatchInfo
{
BatchIndex = 0,
BatchPartsInfo = new List <BatchPartInfo>(new[] { bpi }),
InMemory = false,
Directory = BatchInfo.GenerateNewDirectoryName()
};
}
else
{
batchInfo.BatchPartsInfo.Add(bpi);
}
var bpId = BatchInfo.GenerateNewFileName(httpMessageContent.BatchIndex.ToString());
// to save the file, we should use the local configuration batch directory
var fileName = Path.Combine(this.Configuration.BatchDirectory, batchInfo.Directory, bpId);
BatchPart.Serialize(httpMessageContent.Set, fileName);
bpi.FileName = fileName;
this.LocalProvider.CacheManager.Set("ApplyChanges_BatchInfo", batchInfo);
// Clear the httpMessage set
if (httpMessageContent != null)
{
httpMessageContent.Set.Dispose();
httpMessageContent.Set = null;
}
// if it's last batch sent
if (bpi.IsLastBatch)
{
var(c, s) = await this.ApplyChangesAsync(httpMessage.SyncContext,
new MessageApplyChanges
{
FromScope = scopeInfo,
Schema = schema,
Policy = httpMessageContent.Policy,
UseBulkOperations = httpMessageContent.UseBulkOperations,
ScopeInfoTableName = httpMessageContent.ScopeInfoTableName,
Changes = batchInfo
});
this.LocalProvider.CacheManager.Remove("ApplyChanges_BatchInfo");
httpMessage.SyncContext = c;
httpMessageContent.ChangesApplied = s;
}
httpMessageContent.BatchPartInfo.Clear();
httpMessageContent.BatchPartInfo.FileName = null;
httpMessage.Content = httpMessageContent;
return(httpMessage);
}
public async Task <(SyncContext, BatchInfo, ChangesSelected)> GetChangeBatchAsync(
SyncContext context, MessageGetChangesBatch message)
{
// While we have an other batch to process
var isLastBatch = false;
// Create the BatchInfo and SyncContext to return at the end
BatchInfo changes = new BatchInfo
{
Directory = BatchInfo.GenerateNewDirectoryName()
};
SyncContext syncContext = null;
ChangesSelected changesSelected = null;
while (!isLastBatch)
{
HttpMessage httpMessage = new HttpMessage
{
SyncContext = context,
Step = HttpStep.GetChangeBatch,
Content = new HttpMessageGetChangesBatch
{
ScopeInfo = message.ScopeInfo,
BatchIndexRequested = changes.BatchIndex,
DownloadBatchSizeInKB = message.DownloadBatchSizeInKB,
BatchDirectory = message.BatchDirectory,
Schema = new DmSetSurrogate(message.Schema),
Filters = message.Filters,
Policy = message.Policy,
SerializationFormat = message.SerializationFormat
}
};
var httpMessageResponse = await this.httpRequestHandler.ProcessRequest(httpMessage, message.SerializationFormat, cancellationToken);
if (httpMessageResponse == null)
{
throw new Exception("Can't have an empty body");
}
HttpMessageGetChangesBatch httpMessageContent;
if (httpMessageResponse.Content is HttpMessageGetChangesBatch)
{
httpMessageContent = httpMessageResponse.Content as HttpMessageGetChangesBatch;
}
else
{
httpMessageContent = (httpMessageResponse.Content as JObject).ToObject <HttpMessageGetChangesBatch>();
}
if (httpMessageContent == null)
{
throw new Exception("Can't have an empty GetChangeBatch");
}
changesSelected = httpMessageContent.ChangesSelected;
changes.InMemory = httpMessageContent.InMemory;
syncContext = httpMessageResponse.SyncContext;
// get the bpi and add it to the BatchInfo
var bpi = httpMessageContent.BatchPartInfo;
if (bpi != null)
{
changes.BatchIndex = bpi.Index;
changes.BatchPartsInfo.Add(bpi);
isLastBatch = bpi.IsLastBatch;
}
else
{
changes.BatchIndex = 0;
isLastBatch = true;
// break the while { } story
break;
}
if (changes.InMemory)
{
// load the DmSet in memory
bpi.Set = httpMessageContent.Set.ConvertToDmSet();
}
else
{
// Serialize the file !
var bpId = BatchInfo.GenerateNewFileName(changes.BatchIndex.ToString());
var fileName = Path.Combine(message.BatchDirectory, changes.Directory, bpId);
BatchPart.Serialize(httpMessageContent.Set, fileName);
bpi.FileName = fileName;
bpi.Clear();
}
// Clear the DmSetSurrogate from response, we don't need it anymore
if (httpMessageContent.Set != null)
{
httpMessageContent.Set.Dispose();
httpMessageContent.Set = null;
}
// if not last, increment batchIndex for next request
if (!isLastBatch)
{
changes.BatchIndex++;
}
}
return(syncContext, changes, changesSelected);
}
private async Task <HttpMessage> ApplyChangesAsync(HttpMessage httpMessage)
{
if (httpMessage.ApplyChanges == null)
{
throw new ArgumentException("ApplyChanges message could not be null");
}
var scopeInfo = httpMessage.ApplyChanges.ScopeInfo;
if (scopeInfo == null)
{
throw new ArgumentException("ApplyChanges ScopeInfo could not be null");
}
BatchInfo batchInfo;
var bpi = httpMessage.ApplyChanges.BatchPartInfo;
if (httpMessage.ApplyChanges.InMemory)
{
batchInfo = new BatchInfo
{
BatchIndex = 0,
BatchPartsInfo = new List <BatchPartInfo>(new[] { bpi }),
InMemory = true
};
bpi.Set = httpMessage.ApplyChanges.Set.ConvertToDmSet();
httpMessage.ApplyChanges.Set.Dispose();
httpMessage.ApplyChanges.Set = null;
var(c, s) = await this.ApplyChangesAsync(httpMessage.SyncContext, scopeInfo, batchInfo);
httpMessage.SyncContext = c;
httpMessage.ApplyChanges.ChangesStatistics = s;
httpMessage.ApplyChanges.BatchPartInfo.Clear();
httpMessage.ApplyChanges.BatchPartInfo.FileName = null;
return(httpMessage);
}
// not in memory
batchInfo = this.LocalProvider.CacheManager.GetValue <BatchInfo>("ApplyChanges_BatchInfo");
if (batchInfo == null)
{
batchInfo = new BatchInfo
{
BatchIndex = 0,
BatchPartsInfo = new List <BatchPartInfo>(new[] { bpi }),
InMemory = false,
Directory = BatchInfo.GenerateNewDirectoryName()
};
}
else
{
batchInfo.BatchPartsInfo.Add(bpi);
}
var bpId = BatchInfo.GenerateNewFileName(httpMessage.ApplyChanges.BatchIndex.ToString());
var fileName = Path.Combine(this.LocalProvider.GetCacheConfiguration().BatchDirectory,
batchInfo.Directory, bpId);
BatchPart.Serialize(httpMessage.ApplyChanges.Set, fileName);
bpi.FileName = fileName;
this.LocalProvider.CacheManager.Set("ApplyChanges_BatchInfo", batchInfo);
// Clear the httpMessage set
if (httpMessage.ApplyChanges != null)
{
httpMessage.ApplyChanges.Set.Dispose();
httpMessage.ApplyChanges.Set = null;
}
// if it's last batch sent
if (bpi.IsLastBatch)
{
var(c, s) = await this.ApplyChangesAsync(httpMessage.SyncContext, scopeInfo, batchInfo);
this.LocalProvider.CacheManager.Remove("ApplyChanges_BatchInfo");
httpMessage.SyncContext = c;
httpMessage.ApplyChanges.ChangesStatistics = s;
}
httpMessage.ApplyChanges.BatchPartInfo.Clear();
httpMessage.ApplyChanges.BatchPartInfo.FileName = null;
return(httpMessage);
}
