public void EqualityOnAllColumns()
{
var a = new SlimRow()
{
["id"] = 12, ["name"] = "x",
};
var b = new SlimRow()
{
["id"] = 12, ["name"] = "x",
};
var result = new ColumnBasedRowEqualityComparer().Equals(a, b);
Assert.IsTrue(result);
}
public void EtlRowErrorUnEquality()
{
var a = new SlimRow()
{
["id"] = 12, ["name"] = new EtlRowError("x"),
};
var b = new SlimRow()
{
["id"] = 12, ["name"] = new EtlRowError("y"),
};
var result = new ColumnBasedRowEqualityComparer().Equals(a, b);
Assert.IsFalse(result);
}
public void ReferenceUnEquality()
{
var a = new SlimRow()
{
["person"] = new TestData.PersonModel(),
};
var b = new SlimRow()
{
["person"] = new TestData.PersonModel(),
};
var result = new ColumnBasedRowEqualityComparer().Equals(a, b);
Assert.IsFalse(result);
}
public void StringUnequalityOnAnyColumns()
{
var a = new SlimRow()
{
["id"] = 12, ["name"] = "x",
};
var b = new SlimRow()
{
["id"] = 12, ["name"] = "y",
};
var result = new ColumnBasedRowEqualityComparer().Equals(a, b);
Assert.IsFalse(result);
}
public void ColorUnEquality()
{
var a = new SlimRow()
{
["color"] = Color.Red,
};
var b = new SlimRow()
{
["color"] = Color.Black,
};
var result = new ColumnBasedRowEqualityComparer().Equals(a, b);
Assert.IsFalse(result);
}
public void DoubleUnEquality()
{
var a = new SlimRow()
{
["id"] = -1d,
};
var b = new SlimRow()
{
["id"] = -1.01d,
};
var result = new ColumnBasedRowEqualityComparer().Equals(a, b);
Assert.IsFalse(result);
}
public void DoubleEquality()
{
var a = new SlimRow()
{
["id"] = -6.5d,
};
var b = new SlimRow()
{
["id"] = -13d / 2d,
};
var result = new ColumnBasedRowEqualityComparer().Equals(a, b);
Assert.IsTrue(result);
}
public void IntegerUnEquality()
{
var a = new SlimRow()
{
["id"] = 12,
};
var b = new SlimRow()
{
["id"] = 13,
};
var result = new ColumnBasedRowEqualityComparer().Equals(a, b);
Assert.IsFalse(result);
}
public void EqualityOnSelectedColumns()
{
var a = new SlimRow()
{
["id"] = 12, ["name"] = "x",
};
var b = new SlimRow()
{
["id"] = 12, ["name"] = "y",
};
var result = new ColumnBasedRowEqualityComparer()
{
Columns = new[] { "id" }
}.Equals(a, b);
Assert.IsTrue(result);
}
private static IEnumerable <IMutator> CreateAutoValidityRangeMutators(AutoValidityRangeBuilder builder)
{
var finalValueColumns = builder.CompareValueColumns
.Where(x => !builder.MatchColumns.Contains(x) &&
!x.IsPrimaryKey &&
!builder.PreviousValueColumnNameMap.ContainsValue(x))
.ToArray();
var equalityComparer = new ColumnBasedRowEqualityComparer()
{
Columns = finalValueColumns.Select(x => x.Name).ToArray(),
};
if (builder.MatchColumns.Length == 1)
{
yield return(new BatchedCompareWithRowMutator(builder.TableBuilder.ResilientTable.Topic, nameof(AutoValidityRange))
{
If = row => row.HasValue(builder.MatchColumns[0].Name),
LookupBuilder = new FilteredRowLookupBuilder()
{
ProcessCreator = filterRows => CreateAutoValidity_ExpandDeferredReaderProcess(builder, builder.MatchColumns[0], finalValueColumns, filterRows),
KeyGenerator = row => row.GenerateKey(builder.MatchColumns[0].Name),
},
RowKeyGenerator = row => row.GenerateKey(builder.MatchColumns[0].Name),
EqualityComparer = equalityComparer,
NoMatchAction = new NoMatchAction(MatchMode.Custom)
{
CustomAction = (proc, row) =>
{
// this is the first version
row.SetStagedValue(builder.TableBuilder.ValidFromColumn.Name, builder.TableBuilder.DwhBuilder.DefaultValidFromDateTime);
row.SetStagedValue(builder.TableBuilder.ValidToColumnName, builder.TableBuilder.DwhBuilder.Configuration.InfiniteFutureDateTime);
row.ApplyStaging();
}
},
MatchButDifferentAction = new MatchAction(MatchMode.Custom)
{
CustomAction = (proc, row, match) =>
{
foreach (var kvp in builder.PreviousValueColumnNameMap)
{
var previousValue = match[kvp.Key.Name];
row.SetStagedValue(kvp.Value.Name, previousValue);
}
row.SetStagedValue(builder.TableBuilder.ValidFromColumn.Name, builder.TableBuilder.DwhBuilder.EtlRunIdAsDateTimeOffset.Value);
row.SetStagedValue(builder.TableBuilder.ValidToColumnName, builder.TableBuilder.DwhBuilder.Configuration.InfiniteFutureDateTime);
row.ApplyStaging();
},
},
MatchAndEqualsAction = new MatchAction(MatchMode.Remove)
});
}
else
{
var parameters = new Dictionary <string, object>();
if (builder.TableBuilder.DwhBuilder.Configuration.InfiniteFutureDateTime != null)
{
parameters.Add("InfiniteFuture", builder.TableBuilder.DwhBuilder.Configuration.InfiniteFutureDateTime);
}
yield return(new CompareWithRowMutator(builder.TableBuilder.ResilientTable.Topic, nameof(AutoValidityRange))
{
LookupBuilder = new RowLookupBuilder()
{
Process = new CustomSqlAdoNetDbReader(builder.TableBuilder.ResilientTable.Topic, "PreviousValueReader")
{
ConnectionString = builder.TableBuilder.DwhBuilder.ConnectionString,
MainTableName = builder.TableBuilder.Table.EscapedName(builder.TableBuilder.DwhBuilder.ConnectionString),
Sql = "SELECT " + string.Join(",", builder.MatchColumns.Concat(finalValueColumns).Select(x => x.NameEscaped(builder.TableBuilder.DwhBuilder.ConnectionString)))
+ " FROM " + builder.TableBuilder.Table.EscapedName(builder.TableBuilder.DwhBuilder.ConnectionString)
+ " WHERE " + builder.TableBuilder.ValidToColumnNameEscaped + (builder.TableBuilder.DwhBuilder.Configuration.InfiniteFutureDateTime == null ? " IS NULL" : "=@InfiniteFuture"),
Parameters = parameters,
},
KeyGenerator = row => row.GenerateKey(builder.MatchColumnNames),
},
RowKeyGenerator = row => row.GenerateKey(builder.MatchColumnNames),
EqualityComparer = equalityComparer,
NoMatchAction = new NoMatchAction(MatchMode.Custom)
{
CustomAction = (proc, row) =>
{
// this is the first version
row.SetStagedValue(builder.TableBuilder.ValidFromColumn.Name, builder.TableBuilder.DwhBuilder.DefaultValidFromDateTime);
row.SetStagedValue(builder.TableBuilder.ValidToColumnName, builder.TableBuilder.DwhBuilder.Configuration.InfiniteFutureDateTime);
row.ApplyStaging();
}
},
MatchButDifferentAction = new MatchAction(MatchMode.Custom)
{
CustomAction = (proc, row, match) =>
{
foreach (var kvp in builder.PreviousValueColumnNameMap)
{
var previousValue = match[kvp.Key.Name];
row.SetStagedValue(kvp.Value.Name, previousValue);
}
row.SetStagedValue(builder.TableBuilder.ValidFromColumn.Name, builder.TableBuilder.DwhBuilder.EtlRunIdAsDateTimeOffset.Value);
row.SetStagedValue(builder.TableBuilder.ValidToColumnName, builder.TableBuilder.DwhBuilder.Configuration.InfiniteFutureDateTime);
row.ApplyStaging();
},
},
MatchAndEqualsAction = new MatchAction(MatchMode.Remove)
});
}
}
private static IEnumerable <IMutator> CreateRemoveExistingIdenticalRowsMutators(RemoveExistingRowsBuilder builder)
{
var finalValueColumns = builder.CompareValueColumns?
.Where(x => !builder.MatchColumns.Contains(x) &&
!x.IsPrimaryKey)
.ToArray();
if (finalValueColumns?.Length > 0)
{
var equalityComparer = new ColumnBasedRowEqualityComparer()
{
Columns = finalValueColumns.Select(x => x.Name).ToArray(),
};
if (builder.MatchColumns.Length == 1)
{
yield return(new BatchedCompareWithRowMutator(builder.TableBuilder.ResilientTable.Scope.Context)
{
Name = nameof(RemoveExistingRows),
RowFilter = row => row.HasValue(builder.MatchColumns[0].Name),
EqualityComparer = equalityComparer,
LookupBuilder = new FilteredRowLookupBuilder()
{
ProcessCreator = filterRows => new CustomSqlAdoNetDbReader(builder.TableBuilder.ResilientTable.Scope.Context)
{
Name = "ExistingRowsReader",
ConnectionString = builder.TableBuilder.ResilientTable.Scope.ConnectionString,
MainTableName = builder.TableBuilder.Table.SchemaAndName,
Sql = "SELECT " + builder.MatchColumns[0].NameEscaped(builder.TableBuilder.DwhBuilder.ConnectionString)
+ "," + string.Join(", ", finalValueColumns.Select(c => c.NameEscaped(builder.TableBuilder.DwhBuilder.ConnectionString)))
+ " FROM " + builder.TableBuilder.Table.EscapedName(builder.TableBuilder.DwhBuilder.ConnectionString)
+ " WHERE " + builder.MatchColumns[0].NameEscaped(builder.TableBuilder.DwhBuilder.ConnectionString) + " IN (@keyList)",
InlineArrayParameters = true,
Parameters = new Dictionary <string, object>()
{
["keyList"] = filterRows
.Select(row => row.FormatToString(builder.MatchColumns[0].Name))
.Distinct()
.ToArray(),
},
},
KeyGenerator = row => row.GenerateKey(builder.MatchColumns[0].Name),
},
RowKeyGenerator = row => row.GenerateKey(builder.MatchColumns[0].Name),
MatchAndEqualsAction = new MatchAction(MatchMode.Remove),
MatchButDifferentAction = builder.MatchButDifferentAction,
BatchSize = 2000,
});
}
else
{
yield return(new CompareWithRowMutator(builder.TableBuilder.ResilientTable.Scope.Context)
{
Name = nameof(RemoveExistingRows),
EqualityComparer = equalityComparer,
LookupBuilder = new RowLookupBuilder()
{
Process = new CustomSqlAdoNetDbReader(builder.TableBuilder.ResilientTable.Scope.Context)
{
Name = "ExistingRowsReader",
ConnectionString = builder.TableBuilder.DwhBuilder.ConnectionString,
MainTableName = builder.TableBuilder.Table.SchemaAndName,
Sql = "SELECT " + string.Join(",", builder.MatchColumns.Concat(finalValueColumns).Select(c => c.NameEscaped(builder.TableBuilder.DwhBuilder.ConnectionString)))
+ " FROM " + builder.TableBuilder.Table.EscapedName(builder.TableBuilder.DwhBuilder.ConnectionString),
},
KeyGenerator = row => row.GenerateKey(builder.MatchColumnNames),
},
RowKeyGenerator = row => row.GenerateKey(builder.MatchColumnNames),
MatchAndEqualsAction = new MatchAction(MatchMode.Remove),
MatchButDifferentAction = builder.MatchButDifferentAction,
});
}
}
else if (builder.MatchColumns.Length == 1)
{
yield return(new BatchedKeyTestMutator(builder.TableBuilder.ResilientTable.Scope.Context)
{
Name = nameof(RemoveExistingRows),
RowFilter = row => row.HasValue(builder.MatchColumns[0].Name),
LookupBuilder = new FilteredRowLookupBuilder()
{
ProcessCreator = filterRows => new CustomSqlAdoNetDbReader(builder.TableBuilder.ResilientTable.Scope.Context)
{
Name = "ExistingRowsReader",
ConnectionString = builder.TableBuilder.ResilientTable.Scope.ConnectionString,
MainTableName = builder.TableBuilder.Table.SchemaAndName,
Sql = "SELECT " + builder.MatchColumns[0].NameEscaped(builder.TableBuilder.DwhBuilder.ConnectionString)
+ " FROM " + builder.TableBuilder.Table.EscapedName(builder.TableBuilder.DwhBuilder.ConnectionString)
+ " WHERE " + builder.MatchColumns[0].NameEscaped(builder.TableBuilder.DwhBuilder.ConnectionString) + " IN (@keyList)",
InlineArrayParameters = true,
Parameters = new Dictionary <string, object>()
{
["keyList"] = filterRows
.Select(row => row.FormatToString(builder.MatchColumns[0].Name))
.Distinct()
.ToArray(),
},
},
KeyGenerator = row => row.GenerateKey(builder.MatchColumns[0].Name),
},
RowKeyGenerator = row => row.GenerateKey(builder.MatchColumns[0].Name),
MatchAction = new MatchAction(MatchMode.Remove),
BatchSize = 2000,
});
}
else
{
yield return(new KeyTestMutator(builder.TableBuilder.ResilientTable.Scope.Context)
{
Name = nameof(RemoveExistingRows),
LookupBuilder = new RowLookupBuilder()
{
Process = new CustomSqlAdoNetDbReader(builder.TableBuilder.ResilientTable.Scope.Context)
{
Name = "ExistingRowsReader",
ConnectionString = builder.TableBuilder.DwhBuilder.ConnectionString,
MainTableName = builder.TableBuilder.Table.SchemaAndName,
Sql = "SELECT " + string.Join(",", builder.MatchColumns.Select(c => c.NameEscaped(builder.TableBuilder.DwhBuilder.ConnectionString)))
+ " FROM " + builder.TableBuilder.Table.EscapedName(builder.TableBuilder.DwhBuilder.ConnectionString),
},
KeyGenerator = row => row.GenerateKey(builder.MatchColumnNames),
},
RowKeyGenerator = row => row.GenerateKey(builder.MatchColumnNames),
MatchActionContainsMatch = false,
MatchAction = new MatchAction(MatchMode.Remove),
});
}
}
