private static IEnumerable <IExecutable> CreateSimpleCopyFinalizer(DwhTableBuilder builder)
{
var columnDefaults = new Dictionary <string, object>();
if (builder.HasEtlRunInfo)
{
columnDefaults[builder.EtlRunInsertColumnNameEscaped] = builder.DwhBuilder.EtlRunId.Value;
columnDefaults[builder.EtlRunUpdateColumnNameEscaped] = builder.DwhBuilder.EtlRunId.Value;
}
var columnNames = builder.Table.Columns
.Where(x => !x.GetUsedByEtlRunInfo())
.Select(c => c.NameEscaped(builder.ResilientTable.Scope.Configuration.ConnectionString))
.ToArray();
yield return(new CopyTableIntoExistingTable(builder.ResilientTable.Scope.Context)
{
Name = "CopyToBase",
ConnectionString = builder.ResilientTable.Scope.Configuration.ConnectionString,
Configuration = new TableCopyConfiguration()
{
SourceTableName = builder.ResilientTable.TempTableName,
TargetTableName = builder.ResilientTable.TableName,
Columns = columnNames.ToDictionary(x => x),
},
ColumnDefaults = columnDefaults,
CommandTimeout = 60 * 60,
});
}
private static IEnumerable <IExecutable> CreateDeleteTargetTableFinalizer(DwhTableBuilder builder)
{
builder.ResilientTable.SkipFinalizersIfNoTempData = false;
yield return(new DeleteTable(builder.ResilientTable.Topic, "DeleteBase")
{
ConnectionString = builder.ResilientTable.Scope.Configuration.ConnectionString,
TableName = builder.ResilientTable.TableName,
});
}
private static IEnumerable <IMutator> CreateTrimAllStringColumnLength(DwhTableBuilder builder)
{
var limitedLengthStringColumns = builder.Table.Columns
.Where(x => x.GetLimitedStringLength() != null)
.Select(x => new { column = x, length = x.GetLimitedStringLength().Value, })
.ToList();
if (limitedLengthStringColumns.Count == 0)
{
yield break;
}
yield return(new CustomMutator(builder.ResilientTable.Topic, nameof(TrimAllStringColumnLength))
{
Then = (proc, row) =>
{
foreach (var col in limitedLengthStringColumns)
{
var v = row[col.column.Name];
if (v == null)
{
continue;
}
if (!(v is string strv))
{
continue;
}
if (strv.Length > col.length)
{
var trimv = strv.Substring(0, col.length);
row.SetStagedValue(col.column.Name, trimv);
proc.Context.Log(LogSeverity.Warning, proc, "too long string trimmed on {ConnectionStringName}/{TableName}, column: {Column}, max length: {MaxLength}, original value: {Value}, trimmed value: {TrimValue}",
builder.DwhBuilder.ConnectionString.Name, builder.ResilientTable.TableName, col.column.Name, col.length, strv, trimv);
}
}
row.ApplyStaging();
return true;
}
});
private static IEvaluable CreateSourceTableInputProcess(DwhTableBuilder builder, DateTimeOffset?maxRecordTimestamp, RelationalModel sourceModel, NamedConnectionString sourceConnectionString, AdoNetReaderConnectionScope readerScope, SourceReadSqlStatementCustomizerDelegate sqlStatementCustomizer, string customWhereClause)
{
var whereClauseList = new List <string>();
if (customWhereClause != null)
{
whereClauseList.Add(customWhereClause);
}
var parameterList = new Dictionary <string, object>();
sqlStatementCustomizer?.Invoke(builder, whereClauseList, parameterList);
if (maxRecordTimestamp != null)
{
whereClauseList.Add(builder.Table.GetRecordTimestampIndicatorColumn().NameEscaped(builder.DwhBuilder.ConnectionString) + " >= @MaxRecordTimestamp");
parameterList.Add("MaxRecordTimestamp", maxRecordTimestamp.Value);
}
var sourceTableName = builder.Table.GetSourceTableNameOverride() ?? builder.Table.Name;
var sourceSqlTable = sourceModel[builder.Table.Schema.Name] != null
? sourceModel[builder.Table.Schema.Name][sourceTableName]
: sourceModel.DefaultSchema[sourceTableName];
return(new AdoNetDbReader(builder.ResilientTable.Topic, "SourceTableReader")
{
ConnectionString = sourceConnectionString,
CustomConnectionCreator = readerScope != null ? readerScope.GetConnection : (ConnectionCreatorDelegate)null,
TableName = sourceSqlTable.EscapedName(builder.DwhBuilder.ConnectionString),
CustomWhereClause = whereClauseList.Count == 0
? null
: string.Join(" and ", whereClauseList),
Parameters = parameterList,
ColumnConfiguration = sourceSqlTable.Columns.Select(column =>
new ReaderColumnConfiguration(column.Name, null, /*GetConverter(column.Type.SqlTypeInfo), */ NullSourceHandler.SetSpecialValue, InvalidSourceHandler.WrapError)
).ToList(),
});
}
private static IProducer CreateSourceTableInputProcess(DwhTableBuilder builder, DateTimeOffset?maxRecordTimestamp, RelationalModel sourceModel, NamedConnectionString sourceConnectionString, AdoNetReaderConnectionScope readerScope, SourceReadSqlStatementCustomizerDelegate sqlStatementCustomizer, string customWhereClause)
{
var whereClauseList = new List <string>();
if (customWhereClause != null)
{
whereClauseList.Add(customWhereClause);
}
var parameterList = new Dictionary <string, object>();
sqlStatementCustomizer?.Invoke(builder, whereClauseList, parameterList);
if (maxRecordTimestamp != null)
{
whereClauseList.Add(builder.Table.GetRecordTimestampIndicatorColumn().NameEscaped(builder.DwhBuilder.ConnectionString) + " >= @MaxRecordTimestamp");
parameterList.Add("MaxRecordTimestamp", maxRecordTimestamp.Value);
}
var sourceTableName = builder.Table.GetSourceTableNameOverride() ?? builder.Table.Name;
var sourceSqlTable = sourceModel[builder.Table.Schema.Name] != null
? sourceModel[builder.Table.Schema.Name][sourceTableName]
: sourceModel.DefaultSchema[sourceTableName];
return(new AdoNetDbReader(builder.ResilientTable.Scope.Context)
{
Name = "SourceTableReader",
ConnectionString = sourceConnectionString,
CustomConnectionCreator = readerScope != null ? readerScope.GetConnection : null,
TableName = sourceSqlTable.EscapedName(builder.DwhBuilder.ConnectionString),
CustomWhereClause = whereClauseList.Count == 0
? null
: string.Join(" and ", whereClauseList),
Parameters = parameterList,
Columns = sourceSqlTable.Columns.ToDictionary(column => column.Name, column => new ReaderColumnConfiguration(null)),
});
}
private static IEnumerable <IExecutable> CreateDisableConstraintCheckFinalizer(DwhTableBuilder builder)
{
if (builder.Table.ForeignKeys.Count == 0)
{
yield break;
}
var tableNames = !builder.Table.GetHasHistoryTable()
? new[] { builder.ResilientTable.TableName }
: new[] { builder.ResilientTable.TableName, builder.DwhBuilder.GetEscapedHistTableName(builder.Table) };
yield return(new MsSqlDisableConstraintCheck(builder.ResilientTable.Scope.Context)
{
Name = "DisableConstraintCheck",
ConnectionString = builder.ResilientTable.Scope.Configuration.ConnectionString,
TableNames = tableNames,
CommandTimeout = 60 * 60,
});
yield return(new CustomAction(builder.ResilientTable.Scope.Context)
{
Name = "UpdateConstraintList",
Action = process =>
{
var list = builder.DwhBuilder.Context.AdditionalData.GetAs <List <string> >("ConstraintCheckDisabledOnTables", null);
if (list == null)
{
list = new List <string>();
builder.DwhBuilder.Context.AdditionalData["ConstraintCheckDisabledOnTables"] = list;
}
list.AddRange(tableNames);
}
});
}
internal KeyBasedFinalizerBuilder(DwhTableBuilder tableBuilder)
{
TableBuilder = tableBuilder;
}
internal AutoValidityRangeBuilder(DwhTableBuilder tableBuilder)
{
TableBuilder = tableBuilder;
}
internal RemoveExistingRowsBuilder(DwhTableBuilder tableBuilder)
{
TableBuilder = tableBuilder;
}
