public DataMapStepOutput(DataMap dataMap,
DataTable sourceData, DuplicateRowBehavior sourceDataDuplicateRowBehavior,
DataTable targetData, DuplicateRowBehavior targetDataDuplicateRowBehavior,
Func <DataRow, bool> rowsToProcess, bool deferExecutionUntilNextStep = false)
{
if (dataMap == null)
{
throw new Exception("Data map can not be null.");
}
if (sourceData == null)
{
throw new Exception("Source data can not be null.");
}
if (targetData == null)
{
throw new Exception("Target data can not be null.");
}
DataMap = dataMap;
SourceData = sourceData;
SourceDataDuplicateRowBehavior = sourceDataDuplicateRowBehavior;
TargetData = targetData;
TargetDataDuplicateRowBehavior = targetDataDuplicateRowBehavior;
RowsToProcess = rowsToProcess;
DeferExecutionUntilNextStep = deferExecutionUntilNextStep;
}
public DataMapStepOutput(DataMap dataMap,
DataTable sourceData, DuplicateRowBehavior sourceDataDuplicateRowBehavior,
DataTable targetData, DuplicateRowBehavior targetDataDuplicateRowBehavior,
bool deferExecutionUntilNextStep = false)
: this(dataMap, sourceData, sourceDataDuplicateRowBehavior,
targetData, targetDataDuplicateRowBehavior, null, deferExecutionUntilNextStep)
{
}
/// <summary>
/// Cleans the table of duplicates and returns a collection of keys for removed rows.
/// </summary>
/// <returns></returns>
public static HashSet <RecordKeyCombo> RemoveDuplicates(SyncSide?syncSide, DataTable tbl, List <string> uniqueKeyColumnNames, DuplicateRowBehavior duplicateRowBehavior = DuplicateRowBehavior.ThrowException)
{
if (tbl == null)
{
throw new Exception("Table can not be null.");
}
if (uniqueKeyColumnNames == null || uniqueKeyColumnNames.Count == 0)
{
throw new Exception("At least one unique key column name is required.");
}
if (uniqueKeyColumnNames.Distinct(StringComparer.OrdinalIgnoreCase).Count() < uniqueKeyColumnNames.Count)
{
throw new Exception("All key column names must be unique.");
}
var rowsByKey = new Dictionary <RecordKeyCombo, DataRow>(new RecordKeyComboComparer());
var duplicateRows = new HashSet <DataRow>();
var duplicateKeys = new HashSet <RecordKeyCombo>(new RecordKeyComboComparer());
foreach (DataRow row in tbl.Rows)
{
var rowKeys = GetKeys(row, uniqueKeyColumnNames);
if (rowsByKey.ContainsKey(rowKeys))
{
duplicateKeys.Add(rowKeys);
duplicateRows.Add(row);
if (duplicateRowBehavior == DuplicateRowBehavior.RemoveAllDuplicateRows)
{
duplicateRows.Add(rowsByKey[rowKeys]);
}
}
else
{
rowsByKey[rowKeys] = row;
}
}
if (duplicateRows.Count > 0)
{
if (duplicateRowBehavior == DuplicateRowBehavior.ThrowException)
{
if (syncSide.HasValue)
{
throw new Exception(GetDuplicateRowsExceptionMessage(syncSide.Value, tbl.TableName, uniqueKeyColumnNames, duplicateKeys));
}
else
{
throw new Exception(GetDuplicateRowsExceptionMessage(tbl.TableName, uniqueKeyColumnNames, duplicateKeys));
}
}
else
{
SyncEngineLogger.WriteByParallelTaskContext(LogEntryType.Warning, () =>
{
var msg = new StringBuilder();
if (syncSide.HasValue)
{
msg.Append(GetDuplicateRowsExceptionMessage(syncSide.Value, tbl.TableName, uniqueKeyColumnNames, duplicateKeys));
}
else
{
msg.Append(GetDuplicateRowsExceptionMessage(tbl.TableName, uniqueKeyColumnNames, duplicateKeys));
}
if (duplicateRowBehavior == DuplicateRowBehavior.RemoveAllDuplicateRows)
{
msg.Append(" All duplicate rows were removed.");
}
else if (duplicateRowBehavior == DuplicateRowBehavior.RemoveDuplicateRowsExceptFirst)
{
msg.Append(" All duplicate rows were removed except for the first row for each set of unique keys.");
}
return(msg.ToString());
});
}
}
foreach (var duplicateRow in duplicateRows)
{
tbl.Rows.Remove(duplicateRow);
}
return(duplicateKeys);
}
/// <summary>
/// Cleans the table of duplicates and returns a collection of keys for removed rows.
/// </summary>
/// <returns></returns>
public static HashSet <RecordKeyCombo> RemoveDuplicates(SyncSide?syncSide, DataTable tbl, string uniqueKeyColumnName, DuplicateRowBehavior duplicateRowBehavior = DuplicateRowBehavior.ThrowException)
{
return(RemoveDuplicates(syncSide, tbl, new List <string>()
{
uniqueKeyColumnName
}, duplicateRowBehavior));
}
/// <summary>
/// Cleans the table of duplicates and returns a collection of keys for removed rows.
/// </summary>
/// <returns></returns>
public static HashSet <RecordKeyCombo> RemoveDuplicates(DataTable tbl, List <string> uniqueKeyColumnNames, DuplicateRowBehavior duplicateRowBehavior = DuplicateRowBehavior.ThrowException)
{
return(RemoveDuplicates(null, tbl, uniqueKeyColumnNames, duplicateRowBehavior));
}
public static EntityBatch Compare(OneToMany_OneWayDataMap map,
DataTable sourceData, DuplicateRowBehavior sourceDataDuplicateRowBehavior,
DataTable targetData, DuplicateRowBehavior targetDataDuplicateRowBehavior)
{
if (map == null)
{
throw new Exception("Data map can not be null.");
}
if (sourceData == null)
{
throw new Exception("Source-side data table can not be null.");
}
if (targetData == null)
{
throw new Exception("Target-side data table can not be null.");
}
if (map.EntityToUpdateDefinition.InsertsToExcludeFilter != null)
{
throw new Exception("Exclusion filter for inserts is not supported for one-to-many data maps.");
}
if (map.EntityToUpdateDefinition.UpdatesToExcludeFilter != null)
{
throw new Exception("Exclusion filter for updates is not supported for one-to-many data maps.");
}
if (map.EntityToUpdateDefinition.DeletionsToExcludeFilter != null)
{
throw new Exception("Exclusion filter for deletions is not supported for one-to-many data maps.");
}
if (map.SyncDirection == SyncDirection.SourceToTarget)
{
DataTableHelper.RemoveDuplicates(sourceData, map.JoinKeysCollection.JoinFields.Select(d => d.SourceJoinField).ToList(), sourceDataDuplicateRowBehavior);
}
else if (map.SyncDirection == SyncDirection.TargetToSource)
{
DataTableHelper.RemoveDuplicates(targetData, map.JoinKeysCollection.JoinFields.Select(d => d.TargetJoinField).ToList(), targetDataDuplicateRowBehavior);
}
else
{
throw new EnumValueNotImplementedException <SyncDirection>(map.SyncDirection);
}
//ValidateSyncFieldsExistInDataTables(map, sourceData, targetData);
//ValidateDataOnlyFieldsExistInMap(map);
DataTable oneSideData = null;
List <string> oneSideJoinFields = null;
DataTable manySideData = null;
List <string> manySideJoinFields = null;
var batch = new EntityBatch(map.EntityToUpdateDefinition);
if (map.EntityToUpdateDefinition.SyncSide == SyncSide.Source)
{
if (map.SyncDirection == SyncDirection.SourceToTarget)
{
throw new Exception(string.Format("{0}-side can not be updated for sync direction '{1}'.",
Enum.GetName(typeof(SyncSide), map.EntityToUpdateDefinition.SyncSide),
Enum.GetName(typeof(SyncDirection), map.SyncDirection)));
}
ValidateFieldsToTransposeExistInDataTable(targetData, map.ColumnNamesToTranspose);
oneSideData = targetData;
oneSideJoinFields = map.JoinKeysCollection.JoinFields.Select(d => d.TargetJoinField).ToList();
manySideData = sourceData;
manySideJoinFields = map.JoinKeysCollection.JoinFields.Select(d => d.SourceJoinField).ToList();
}
else if (map.EntityToUpdateDefinition.SyncSide == SyncSide.Target)
{
if (map.SyncDirection == SyncDirection.TargetToSource)
{
throw new Exception(string.Format("{0}-side can not be updated for sync direction '{1}'.",
Enum.GetName(typeof(SyncSide), map.EntityToUpdateDefinition.SyncSide),
Enum.GetName(typeof(SyncDirection), map.SyncDirection)));
}
ValidateFieldsToTransposeExistInDataTable(sourceData, map.ColumnNamesToTranspose);
oneSideData = sourceData;
oneSideJoinFields = map.JoinKeysCollection.JoinFields.Select(d => d.SourceJoinField).ToList();
manySideData = targetData;
manySideJoinFields = map.JoinKeysCollection.JoinFields.Select(d => d.TargetJoinField).ToList();
}
else
{
throw new EnumValueNotImplementedException <SyncSide>(map.EntityToUpdateDefinition.SyncSide);
}
var customSetFieldsForDelete = map.CustomSetFields
.Where(d => (d.AppliesTo.HasFlag(SyncOperation.Deletes) || d.AppliesTo.HasFlag(SyncOperation.All)));
var dataOnlyFields = batch.EntityDefinition.DataOnlyFields.ToDictionary(d => d.FieldName, d => d, StringComparer.OrdinalIgnoreCase);
var manySideRowsLookup = GetManySideRowsDictionary(manySideJoinFields, manySideData);
foreach (var oneSideRow in oneSideData.Rows.Cast <DataRow>())
{
var oneSideRecordKeyCombo = RecordKeyCombo.GetRecordKeyComboFromDataRow(oneSideRow, oneSideJoinFields);
List <DataRow> manySideFilteredRows = null;
if (manySideRowsLookup.ContainsKey(oneSideRecordKeyCombo))
{
manySideFilteredRows = manySideRowsLookup[oneSideRecordKeyCombo];
}
else
{
manySideFilteredRows = new List <DataRow>();
}
foreach (var fieldToTranspose in map.ColumnNamesToTranspose)
{
var transposeResult = map.TransposeMethod(manySideFilteredRows, oneSideRow, fieldToTranspose);
if (transposeResult == null)
{
continue;
}
var transposeDataOnlyValues = GetTransposeDataOnlyFieldValues(map.EntityToUpdateDefinition.TransposeDataOnlyFields, oneSideRow, manySideFilteredRows, fieldToTranspose);
EntityRecord entityRecord;
if (transposeResult is TransposeResult_AddRecord && batch.EntityDefinition.ApplyInserts)
{
entityRecord = AddInsertToBatch(map, batch, dataOnlyFields, oneSideRow, transposeResult, transposeDataOnlyValues);
}
else if (transposeResult is TransposeResult_UpdateRecord && batch.EntityDefinition.ApplyUpdates)
{
entityRecord = AddUpdateToBatch(map, batch, dataOnlyFields, oneSideRow, manySideFilteredRows, transposeResult, transposeDataOnlyValues);
}
else if (transposeResult is TransposeResult_DeleteRecord && batch.EntityDefinition.ApplyDeletions)
{
entityRecord = AddDeletionToBatch(map, batch, dataOnlyFields, oneSideRow, manySideFilteredRows, transposeResult, transposeDataOnlyValues);
}
else
{
throw new DerivedClassNotImplementedException <TransposeResult>(transposeResult);
}
}
}
return(batch);
}
