/// <summary>
/// Data load with delta changes
/// </summary>
private void DeltaLoad()
{
// Log into Load Status table and set delta version
int loadId = businessLayer.InsertLoadStatusAndReturnId();
// get the tables with valid delta version
var validDeltaTables = businessLayer.GetTablesWithValidDeltaChanges();
// ready to load the data
businessLayer.UpdateLoadStatusLog((char)LoadStatus.Preparing, (char)LoadStatusType.Delta, loadId);
try
{
// 2. Start processing data for Historic load into replica database
Parallel.ForEach(entities, new ParallelOptions() { MaxDegreeOfParallelism = maxParallelThread }, (entity) =>
{
if (businessLayer.IsTableExists(entity, CloudConfigurationManager.GetSetting("TargetDatabaseConnectionString")))
{
// pull all the data into in-memory datatable
DataTable entityData = new DataTable();
this.TruncateEntityData(entity);
Trace.WriteLine(string.Format("Processing data for entity {0}", entity));
if (transactionTables.Contains(entity) && validDeltaTables.Contains(entity))
{
// get delta changes
entityData = businessLayer.GetTableDataWithDeltaChanges(entity);
}
// Reference tables need to copy with full data
// The table set should be mutually exclusive between Reference tables and Transaction tables
if(referenceTables.Contains(entity))
{
// get full load, no valid delta for this entity
entityData = businessLayer.GetEntityWithData(entity);
}
// decrypt the account data
if (entity == "Accounts" && entityData.Rows.Count > 0 && CloudConfigurationManager.GetSetting("IsAccountDecryptionReqd") == "1")
{
entityData = businessLayer.DecryptAccountRecords(entityData, CloudConfigurationManager.GetSetting("EncryptionCert"));
}
if (entityData.Rows.Count > 0)
{
// write data to destination table
BulkWriter tableWriter = new BulkWriter(entity, businessLayer.GetColumns(entity, entity));
tableWriter.WriteWithRetries(entityData);
Trace.WriteLine(string.Format("Data written for entity {0}", entity));
}
}
});
}
catch (OutOfMemoryException ex)
{
this.LogError(ex, loadId);
}
catch (ApplicationException ex)
{
this.LogError(ex, loadId);
}
catch (SqlException ex)
{
this.LogError(ex, loadId);
}
catch (Exception ex)
{
this.LogError(ex, loadId);
}
// 3. Update the load_status_details table
businessLayer.UpdateLoadStatusLog(
(char)LoadStatus.Ready,
validDeltaTables.Count > 0 ? (char)LoadStatusType.Delta : (char)LoadStatusType.Historic,
loadId);
}
/// <summary>
/// Initial Load from Robbins database to Robbins replica
/// </summary>
private void InitialLoad()
{
// Log into Load Status table
int loadId = businessLayer.InsertLoadStatusAndReturnId();
// ready to load the data
businessLayer.UpdateLoadStatusLog((char)LoadStatus.Preparing, (char)LoadStatusType.Historic, loadId);
try
{
// 2. Start processing data for Historic load into replica database
Parallel.ForEach(entities, new ParallelOptions() { MaxDegreeOfParallelism = maxParallelThread }, (entity) =>
{
// truncate all the existing data to avoid duplication
if (businessLayer.IsTableExists(entity, CloudConfigurationManager.GetSetting("TargetDatabaseConnectionString")) && fullLoadTables.Contains(entity))
{
Trace.WriteLine(string.Format("Truncating data for entity {0}", entity));
businessLayer.TruncateEntity(entity);
Trace.WriteLine(string.Format("Processing data for entity {0}", entity));
// pull all the data into in-memory datatable
var entityData = businessLayer.GetEntityWithData(entity);
if (entity == "Accounts" && CloudConfigurationManager.GetSetting("IsAccountDecryptionReqd")=="1")
{
entityData = businessLayer.DecryptAccountRecords(entityData, CloudConfigurationManager.GetSetting("EncryptionCert"));
}
// write data to destination table
BulkWriter tableWriter = new BulkWriter(entity, businessLayer.GetColumns(entity, entity));
tableWriter.WriteWithRetries(entityData);
Trace.WriteLine(string.Format("Data written for entity {0}", entity));
}
});
}
catch(OutOfMemoryException ex)
{
this.LogError(ex, loadId);
}
catch(ApplicationException ex)
{
this.LogError(ex, loadId);
}
catch(SqlException ex)
{
this.LogError(ex, loadId);
}
catch (Exception ex)
{
this.LogError(ex, loadId);
}
// 3. Update the load_status_details table
businessLayer.UpdateLoadStatusLog((char)LoadStatus.Ready, (char)LoadStatusType.Historic, loadId);
}
