private static IEnumerable <IExecutable> CreateSimpleCopyFinalizer(DwhTableBuilder builder) { var columnDefaults = new Dictionary <string, object>(); if (builder.HasEtlRunInfo) { columnDefaults[builder.EtlRunInsertColumnNameEscaped] = builder.DwhBuilder.EtlRunId.Value; columnDefaults[builder.EtlRunUpdateColumnNameEscaped] = builder.DwhBuilder.EtlRunId.Value; } var columnNames = builder.Table.Columns .Where(x => !x.GetUsedByEtlRunInfo()) .Select(c => c.NameEscaped(builder.ResilientTable.Scope.Configuration.ConnectionString)) .ToArray(); yield return(new CopyTableIntoExistingTable(builder.ResilientTable.Scope.Context) { Name = "CopyToBase", ConnectionString = builder.ResilientTable.Scope.Configuration.ConnectionString, Configuration = new TableCopyConfiguration() { SourceTableName = builder.ResilientTable.TempTableName, TargetTableName = builder.ResilientTable.TableName, Columns = columnNames.ToDictionary(x => x), }, ColumnDefaults = columnDefaults, CommandTimeout = 60 * 60, }); }
private static IEnumerable <IExecutable> CreateDeleteTargetTableFinalizer(DwhTableBuilder builder) { builder.ResilientTable.SkipFinalizersIfNoTempData = false; yield return(new DeleteTable(builder.ResilientTable.Topic, "DeleteBase") { ConnectionString = builder.ResilientTable.Scope.Configuration.ConnectionString, TableName = builder.ResilientTable.TableName, }); }
private static IEnumerable <IMutator> CreateTrimAllStringColumnLength(DwhTableBuilder builder) { var limitedLengthStringColumns = builder.Table.Columns .Where(x => x.GetLimitedStringLength() != null) .Select(x => new { column = x, length = x.GetLimitedStringLength().Value, }) .ToList(); if (limitedLengthStringColumns.Count == 0) { yield break; } yield return(new CustomMutator(builder.ResilientTable.Topic, nameof(TrimAllStringColumnLength)) { Then = (proc, row) => { foreach (var col in limitedLengthStringColumns) { var v = row[col.column.Name]; if (v == null) { continue; } if (!(v is string strv)) { continue; } if (strv.Length > col.length) { var trimv = strv.Substring(0, col.length); row.SetStagedValue(col.column.Name, trimv); proc.Context.Log(LogSeverity.Warning, proc, "too long string trimmed on {ConnectionStringName}/{TableName}, column: {Column}, max length: {MaxLength}, original value: {Value}, trimmed value: {TrimValue}", builder.DwhBuilder.ConnectionString.Name, builder.ResilientTable.TableName, col.column.Name, col.length, strv, trimv); } } row.ApplyStaging(); return true; } });
private static IEvaluable CreateSourceTableInputProcess(DwhTableBuilder builder, DateTimeOffset?maxRecordTimestamp, RelationalModel sourceModel, NamedConnectionString sourceConnectionString, AdoNetReaderConnectionScope readerScope, SourceReadSqlStatementCustomizerDelegate sqlStatementCustomizer, string customWhereClause) { var whereClauseList = new List <string>(); if (customWhereClause != null) { whereClauseList.Add(customWhereClause); } var parameterList = new Dictionary <string, object>(); sqlStatementCustomizer?.Invoke(builder, whereClauseList, parameterList); if (maxRecordTimestamp != null) { whereClauseList.Add(builder.Table.GetRecordTimestampIndicatorColumn().NameEscaped(builder.DwhBuilder.ConnectionString) + " >= @MaxRecordTimestamp"); parameterList.Add("MaxRecordTimestamp", maxRecordTimestamp.Value); } var sourceTableName = builder.Table.GetSourceTableNameOverride() ?? builder.Table.Name; var sourceSqlTable = sourceModel[builder.Table.Schema.Name] != null ? sourceModel[builder.Table.Schema.Name][sourceTableName] : sourceModel.DefaultSchema[sourceTableName]; return(new AdoNetDbReader(builder.ResilientTable.Topic, "SourceTableReader") { ConnectionString = sourceConnectionString, CustomConnectionCreator = readerScope != null ? readerScope.GetConnection : (ConnectionCreatorDelegate)null, TableName = sourceSqlTable.EscapedName(builder.DwhBuilder.ConnectionString), CustomWhereClause = whereClauseList.Count == 0 ? null : string.Join(" and ", whereClauseList), Parameters = parameterList, ColumnConfiguration = sourceSqlTable.Columns.Select(column => new ReaderColumnConfiguration(column.Name, null, /*GetConverter(column.Type.SqlTypeInfo), */ NullSourceHandler.SetSpecialValue, InvalidSourceHandler.WrapError) ).ToList(), }); }
private static IProducer CreateSourceTableInputProcess(DwhTableBuilder builder, DateTimeOffset?maxRecordTimestamp, RelationalModel sourceModel, NamedConnectionString sourceConnectionString, AdoNetReaderConnectionScope readerScope, SourceReadSqlStatementCustomizerDelegate sqlStatementCustomizer, string customWhereClause) { var whereClauseList = new List <string>(); if (customWhereClause != null) { whereClauseList.Add(customWhereClause); } var parameterList = new Dictionary <string, object>(); sqlStatementCustomizer?.Invoke(builder, whereClauseList, parameterList); if (maxRecordTimestamp != null) { whereClauseList.Add(builder.Table.GetRecordTimestampIndicatorColumn().NameEscaped(builder.DwhBuilder.ConnectionString) + " >= @MaxRecordTimestamp"); parameterList.Add("MaxRecordTimestamp", maxRecordTimestamp.Value); } var sourceTableName = builder.Table.GetSourceTableNameOverride() ?? builder.Table.Name; var sourceSqlTable = sourceModel[builder.Table.Schema.Name] != null ? sourceModel[builder.Table.Schema.Name][sourceTableName] : sourceModel.DefaultSchema[sourceTableName]; return(new AdoNetDbReader(builder.ResilientTable.Scope.Context) { Name = "SourceTableReader", ConnectionString = sourceConnectionString, CustomConnectionCreator = readerScope != null ? readerScope.GetConnection : null, TableName = sourceSqlTable.EscapedName(builder.DwhBuilder.ConnectionString), CustomWhereClause = whereClauseList.Count == 0 ? null : string.Join(" and ", whereClauseList), Parameters = parameterList, Columns = sourceSqlTable.Columns.ToDictionary(column => column.Name, column => new ReaderColumnConfiguration(null)), }); }
private static IEnumerable <IExecutable> CreateDisableConstraintCheckFinalizer(DwhTableBuilder builder) { if (builder.Table.ForeignKeys.Count == 0) { yield break; } var tableNames = !builder.Table.GetHasHistoryTable() ? new[] { builder.ResilientTable.TableName } : new[] { builder.ResilientTable.TableName, builder.DwhBuilder.GetEscapedHistTableName(builder.Table) }; yield return(new MsSqlDisableConstraintCheck(builder.ResilientTable.Scope.Context) { Name = "DisableConstraintCheck", ConnectionString = builder.ResilientTable.Scope.Configuration.ConnectionString, TableNames = tableNames, CommandTimeout = 60 * 60, }); yield return(new CustomAction(builder.ResilientTable.Scope.Context) { Name = "UpdateConstraintList", Action = process => { var list = builder.DwhBuilder.Context.AdditionalData.GetAs <List <string> >("ConstraintCheckDisabledOnTables", null); if (list == null) { list = new List <string>(); builder.DwhBuilder.Context.AdditionalData["ConstraintCheckDisabledOnTables"] = list; } list.AddRange(tableNames); } }); }
internal KeyBasedFinalizerBuilder(DwhTableBuilder tableBuilder) { TableBuilder = tableBuilder; }
internal AutoValidityRangeBuilder(DwhTableBuilder tableBuilder) { TableBuilder = tableBuilder; }
internal RemoveExistingRowsBuilder(DwhTableBuilder tableBuilder) { TableBuilder = tableBuilder; }