/// <summary> /// Data load with delta changes /// </summary> private void DeltaLoad() { // Log into Load Status table and set delta version int loadId = businessLayer.InsertLoadStatusAndReturnId(); // get the tables with valid delta version var validDeltaTables = businessLayer.GetTablesWithValidDeltaChanges(); // ready to load the data businessLayer.UpdateLoadStatusLog((char)LoadStatus.Preparing, (char)LoadStatusType.Delta, loadId); try { // 2. Start processing data for Historic load into replica database Parallel.ForEach(entities, new ParallelOptions() { MaxDegreeOfParallelism = maxParallelThread }, (entity) => { if (businessLayer.IsTableExists(entity, CloudConfigurationManager.GetSetting("TargetDatabaseConnectionString"))) { // pull all the data into in-memory datatable DataTable entityData = new DataTable(); this.TruncateEntityData(entity); Trace.WriteLine(string.Format("Processing data for entity {0}", entity)); if (transactionTables.Contains(entity) && validDeltaTables.Contains(entity)) { // get delta changes entityData = businessLayer.GetTableDataWithDeltaChanges(entity); } // Reference tables need to copy with full data // The table set should be mutually exclusive between Reference tables and Transaction tables if(referenceTables.Contains(entity)) { // get full load, no valid delta for this entity entityData = businessLayer.GetEntityWithData(entity); } // decrypt the account data if (entity == "Accounts" && entityData.Rows.Count > 0 && CloudConfigurationManager.GetSetting("IsAccountDecryptionReqd") == "1") { entityData = businessLayer.DecryptAccountRecords(entityData, CloudConfigurationManager.GetSetting("EncryptionCert")); } if (entityData.Rows.Count > 0) { // write data to destination table BulkWriter tableWriter = new BulkWriter(entity, businessLayer.GetColumns(entity, entity)); tableWriter.WriteWithRetries(entityData); Trace.WriteLine(string.Format("Data written for entity {0}", entity)); } } }); } catch (OutOfMemoryException ex) { this.LogError(ex, loadId); } catch (ApplicationException ex) { this.LogError(ex, loadId); } catch (SqlException ex) { this.LogError(ex, loadId); } catch (Exception ex) { this.LogError(ex, loadId); } // 3. Update the load_status_details table businessLayer.UpdateLoadStatusLog( (char)LoadStatus.Ready, validDeltaTables.Count > 0 ? (char)LoadStatusType.Delta : (char)LoadStatusType.Historic, loadId); }
/// <summary> /// Initial Load from Robbins database to Robbins replica /// </summary> private void InitialLoad() { // Log into Load Status table int loadId = businessLayer.InsertLoadStatusAndReturnId(); // ready to load the data businessLayer.UpdateLoadStatusLog((char)LoadStatus.Preparing, (char)LoadStatusType.Historic, loadId); try { // 2. Start processing data for Historic load into replica database Parallel.ForEach(entities, new ParallelOptions() { MaxDegreeOfParallelism = maxParallelThread }, (entity) => { // truncate all the existing data to avoid duplication if (businessLayer.IsTableExists(entity, CloudConfigurationManager.GetSetting("TargetDatabaseConnectionString")) && fullLoadTables.Contains(entity)) { Trace.WriteLine(string.Format("Truncating data for entity {0}", entity)); businessLayer.TruncateEntity(entity); Trace.WriteLine(string.Format("Processing data for entity {0}", entity)); // pull all the data into in-memory datatable var entityData = businessLayer.GetEntityWithData(entity); if (entity == "Accounts" && CloudConfigurationManager.GetSetting("IsAccountDecryptionReqd")=="1") { entityData = businessLayer.DecryptAccountRecords(entityData, CloudConfigurationManager.GetSetting("EncryptionCert")); } // write data to destination table BulkWriter tableWriter = new BulkWriter(entity, businessLayer.GetColumns(entity, entity)); tableWriter.WriteWithRetries(entityData); Trace.WriteLine(string.Format("Data written for entity {0}", entity)); } }); } catch(OutOfMemoryException ex) { this.LogError(ex, loadId); } catch(ApplicationException ex) { this.LogError(ex, loadId); } catch(SqlException ex) { this.LogError(ex, loadId); } catch (Exception ex) { this.LogError(ex, loadId); } // 3. Update the load_status_details table businessLayer.UpdateLoadStatusLog((char)LoadStatus.Ready, (char)LoadStatusType.Historic, loadId); }