private static IEnumerable <IJob> CreateDisableConstraintCheckFinalizer(DwhTableBuilder builder)
{
if (builder.Table.ForeignKeys.Count == 0)
{
yield break;
}
var tableNames = !builder.Table.GetHasHistoryTable()
? new[] { builder.ResilientTable.TableName }
: new[] { builder.ResilientTable.TableName, builder.DwhBuilder.GetEscapedHistTableName(builder.Table) };
yield return(new MsSqlDisableConstraintCheck(builder.ResilientTable.Scope.Context)
{
Name = "DisableConstraintCheck",
ConnectionString = builder.ResilientTable.Scope.ConnectionString,
TableNames = tableNames,
CommandTimeout = 60 * 60,
});
yield return(new CustomJob(builder.ResilientTable.Scope.Context)
{
Name = "UpdateConstraintList",
Action = process =>
{
var list = builder.DwhBuilder.Context.AdditionalData.GetAs <List <string> >("ConstraintCheckDisabledOnTables", null);
if (list == null)
{
list = new List <string>();
builder.DwhBuilder.Context.AdditionalData["ConstraintCheckDisabledOnTables"] = list;
}
list.AddRange(tableNames);
}
});
}
private static IEnumerable <IJob> CreateSimpleCopyFinalizer(DwhTableBuilder builder)
{
var columnDefaults = new Dictionary <string, object>();
if (builder.HasEtlRunInfo)
{
columnDefaults[builder.EtlRunInsertColumnNameEscaped] = builder.DwhBuilder.EtlRunId.Value;
columnDefaults[builder.EtlRunUpdateColumnNameEscaped] = builder.DwhBuilder.EtlRunId.Value;
}
var columnNames = builder.Table.Columns
.Where(x => !x.GetUsedByEtlRunInfo())
.Select(c => c.NameEscaped(builder.ResilientTable.Scope.ConnectionString))
.ToArray();
yield return(new CopyTableIntoExistingTable(builder.ResilientTable.Scope.Context)
{
Name = "CopyToBase",
ConnectionString = builder.ResilientTable.Scope.ConnectionString,
Configuration = new TableCopyConfiguration()
{
SourceTableName = builder.ResilientTable.TempTableName,
TargetTableName = builder.ResilientTable.TableName,
Columns = columnNames.ToDictionary(x => x),
},
ColumnDefaults = columnDefaults,
CommandTimeout = 60 * 60,
});
}
private static IEnumerable <IMutator> PetMutators(DwhTableBuilder tableBuilder)
{
yield return(new CustomMutator(tableBuilder.ResilientTable.Topic, "FkFix")
{
Then = (proc, row) =>
{
var fk = row.GetAs <int?>("OwnerPeopleId");
return fk != null && fk.Value >= 0;
},
});
}
private static IEnumerable <IJob> CreateDeleteTargetTableFinalizer(DwhTableBuilder builder)
{
builder.ResilientTable.SkipFinalizersIfNoTempData = false;
yield return(new DeleteTable(builder.ResilientTable.Scope.Context)
{
Name = "DeleteBase",
ConnectionString = builder.ResilientTable.Scope.ConnectionString,
TableName = builder.ResilientTable.TableName,
});
}
public static IEvaluable CreatePeopleRating2(DwhTableBuilder tableBuilder, DateTimeOffset?maxRecordTimestamp)
{
return(new RowCreator(tableBuilder.ResilientTable.Topic, null)
{
Columns = new[] { "PeopleId", "Rating" },
InputRows = new List <object[]>()
{
new object[] { 0, 29 },
new object[] { 3, 23 },
},
});
}
public static IEvaluable CreateCompany2(DwhTableBuilder tableBuilder, DateTimeOffset?maxRecordTimestamp)
{
return(new RowCreator(tableBuilder.ResilientTable.Topic, null)
{
Columns = new[] { "Id", "Name" },
InputRows = new List <object[]>()
{
new object[] { 2, "Bx" },
new object[] { 3, "Cx" },
new object[] { 5, "E" },
},
});
}
public static IEvaluable CreatePet2(DwhTableBuilder tableBuilder, DateTimeOffset?maxRecordTimestamp)
{
return(new RowCreator(tableBuilder.ResilientTable.Topic, null)
{
Columns = new[] { "Id", "Name", "OwnerPeopleId", "LastChangedOn" },
InputRows = new List <object[]>()
{
new object[] { 1, "pet#1", 0, new DateTime(2000, 1, 1, 1, 1, 1, DateTimeKind.Utc) },
new object[] { 2, "pet#2x", 0, new DateTime(2010, 1, 1, 1, 1, 1, DateTimeKind.Utc) },
new object[] { 3, "pet#3", 2, new DateTime(2000, 1, 1, 1, 1, 1, DateTimeKind.Utc) },
new object[] { 4, "pet#4x", 0, new DateTime(2010, 1, 1, 1, 1, 1, DateTimeKind.Utc) },
new object[] { 5, "pet#5", -1, new DateTime(2000, 1, 1, 1, 1, 1, DateTimeKind.Utc) },
},
});
}
public static IEvaluable CreatePeople2(DwhTableBuilder tableBuilder, DateTimeOffset?maxRecordTimestamp)
{
return(new RowCreator(tableBuilder.ResilientTable.Topic, null)
{
Columns = new[] { "Id", "Name", "FavoritePetId", "LastChangedOn" },
InputRows = new List <object[]>()
{
new object[] { 0, "A", 2, new DateTime(2010, 1, 1, 1, 1, 1, DateTimeKind.Utc) },
new object[] { 1, "Bx", null, new DateTime(2010, 1, 1, 1, 1, 1, DateTimeKind.Utc) },
new object[] { 2, "C", 3, new DateTime(2000, 1, 1, 1, 1, 1, DateTimeKind.Utc) },
new object[] { 3, "Dx", null, new DateTime(2010, 1, 1, 1, 1, 1, DateTimeKind.Utc) },
new object[] { 4, "E", null, new DateTime(2000, 1, 1, 1, 1, 1, DateTimeKind.Utc) },
new object[] { 5, "Fx", -1, new DateTime(2010, 1, 1, 1, 1, 1, DateTimeKind.Utc) },
},
});
}
private static IEnumerable <IMutator> CreateTrimAllStringColumnLength(DwhTableBuilder builder)
{
var limitedLengthStringColumns = builder.Table.Columns
.Where(x => x.GetLimitedStringLength() != null)
.Select(x => new { column = x, length = x.GetLimitedStringLength().Value, })
.ToList();
if (limitedLengthStringColumns.Count == 0)
{
yield break;
}
yield return(new CustomMutator(builder.ResilientTable.Scope.Context)
{
Name = nameof(TrimAllStringColumnLength),
Action = row =>
{
foreach (var col in limitedLengthStringColumns)
{
var v = row[col.column.Name];
if (v == null)
{
continue;
}
if (v is not string strv)
{
continue;
}
if (strv.Length > col.length)
{
var trimv = strv.Substring(0, col.length);
row[col.column.Name] = trimv;
row.CurrentProcess.Context.Log(LogSeverity.Warning, row.CurrentProcess, "too long string trimmed on {ConnectionStringName}/{TableName}, column: {Column}, max length: {MaxLength}, original value: {Value}, trimmed value: {TrimValue}",
builder.DwhBuilder.ConnectionString.Name, builder.ResilientTable.TableName, col.column.Name, col.length, strv, trimv);
}
}
return true;
}
});
private static ISequence CreateSourceTableInputProcess(DwhTableBuilder builder, DateTimeOffset?maxRecordTimestamp, RelationalModel sourceModel, NamedConnectionString sourceConnectionString, AdoNetReaderConnectionScope readerScope, SourceReadSqlStatementCustomizerDelegate sqlStatementCustomizer, string customWhereClause)
{
var whereClauseList = new List <string>();
if (customWhereClause != null)
{
whereClauseList.Add(customWhereClause);
}
var parameterList = new Dictionary <string, object>();
sqlStatementCustomizer?.Invoke(builder, whereClauseList, parameterList);
if (maxRecordTimestamp != null)
{
whereClauseList.Add(builder.Table.GetRecordTimestampIndicatorColumn().NameEscaped(builder.DwhBuilder.ConnectionString) + " >= @MaxRecordTimestamp");
parameterList.Add("MaxRecordTimestamp", maxRecordTimestamp.Value);
}
var sourceTableName = builder.Table.GetSourceTableNameOverride() ?? builder.Table.Name;
var sourceSqlTable = sourceModel[builder.Table.Schema.Name] != null
? sourceModel[builder.Table.Schema.Name][sourceTableName]
: sourceModel.DefaultSchema[sourceTableName];
return(new AdoNetDbReader(builder.ResilientTable.Scope.Context)
{
Name = "SourceTableReader",
ConnectionString = sourceConnectionString,
CustomConnectionCreator = readerScope != null ? readerScope.GetConnection : null,
TableName = sourceSqlTable.EscapedName(builder.DwhBuilder.ConnectionString),
CustomWhereClause = whereClauseList.Count == 0
? null
: string.Join(" and ", whereClauseList),
Parameters = parameterList,
Columns = sourceSqlTable.Columns.ToDictionary(column => column.Name, column => new ReaderColumn(null)),
});
}
internal RemoveExistingRowsBuilder(DwhTableBuilder tableBuilder)
{
TableBuilder = tableBuilder;
}
internal KeyBasedFinalizerBuilder(DwhTableBuilder tableBuilder)
{
TableBuilder = tableBuilder;
}
internal AutoValidityRangeBuilder(DwhTableBuilder tableBuilder)
{
TableBuilder = tableBuilder;
}