public static void ProcessSlowlyChangingDimensionTransformations(SymbolTable symbolTable) ////HashSet<AstEtlRootNode> astEtlRootNodes, HashSet<AstTaskNode> astTaskNodes)
{
var snapshotSymbolTable = new List <IReferenceableItem>(symbolTable);
foreach (var astNamedNode in snapshotSymbolTable)
{
if (astNamedNode.FirstThisOrParent <ITemplate>() == null)
{
var destination = astNamedNode as AstDestinationNode;
if (destination != null)
{
if (destination.Table.HasScdColumns && !destination.DisableScd)
{
Utility.Replace(destination, CreateScdWorkflowFragment(destination.Table, destination.ParentItem, destination.InputPath == null ? null : destination.InputPath.OutputPath));
}
}
var merge = astNamedNode as AstMergeTaskNode;
if (merge != null)
{
var sourceTable = merge.TargetConstraint.ParentItem as AstTableNode;
// TODO: Must we do anything special for the UpdateTargetTable attribute?
if (sourceTable != null && sourceTable.HasScdColumns && !merge.DisableScd)
{
var targetTable = merge.TargetConstraint.ParentItem as AstTableNode;
if (targetTable == null)
{
continue;
}
var scdMergeEtl = new AstEtlRootNode(merge.ParentItem)
{
Name = Utility.NameCleanerAndUniqifier(merge.Name + "_scdEtl")
};
// TODO: Do we need to limit query to non-computed columns?
var scdMergeSource = new AstQuerySourceNode(scdMergeEtl)
{
Connection = sourceTable.Connection,
Name = Utility.NameCleanerAndUniqifier(merge.Name + "_scdEtlSource"),
Query = new AstVariableParameterMappingQueryNode(null)
{
Body = TableLowerer.EmitSelectAllStatement(sourceTable)
}
};
scdMergeSource.Query.ParentItem = scdMergeSource;
scdMergeEtl.Transformations.Add(scdMergeSource);
foreach (var transformation in CreateScdWorkflowFragment(targetTable, scdMergeEtl, scdMergeSource.OutputPath))
{
scdMergeEtl.Transformations.Add(transformation);
}
}
}
}
}
}
private static AstLookupNode CreateLookupNode(AstLateArrivingLookupNode lookup)
{
var codegenLookup = new AstLookupNode(lookup.ParentItem)
{
Name = String.Format(CultureInfo.InvariantCulture, "__LALookupEntryPoint_{0}", lookup.Name),
Connection = lookup.Table.Connection
};
var requiredColumns = new List <string>();
foreach (var input in lookup.Inputs)
{
if (!requiredColumns.Contains(input.RemoteColumnName))
{
requiredColumns.Add(input.RemoteColumnName);
}
codegenLookup.Inputs.Add(input);
}
foreach (var output in lookup.Outputs)
{
if (!requiredColumns.Contains(output.RemoteColumnName))
{
requiredColumns.Add(output.RemoteColumnName);
}
codegenLookup.Outputs.Add(output);
}
codegenLookup.Query = new AstQueryNode(codegenLookup)
{
Body = TableLowerer.EmitSelectAllStatement(lookup.Table, requiredColumns)
};
if (lookup.InputPath != null)
{
codegenLookup.InputPath = new AstDataflowMappedInputPathNode(codegenLookup)
{
OutputPath = lookup.InputPath.OutputPath
};
}
return(codegenLookup);
}
private static List <AstTransformationNode> CreateScdWorkflowFragment(AstTableNode targetTable, IFrameworkItem parentItem, AstDataflowOutputPathNode outputPath)
{
var workflowFragment = new List <AstTransformationNode>();
AstTableColumnBaseNode lateArrivingStatusColumn = targetTable.Columns.FirstOrDefault(item => item.Name == "_IsLate");
var scd = new AstSlowlyChangingDimensionNode(parentItem);
scd.Name = Utility.NameCleanerAndUniqifier(targetTable.Name + "_scd");
scd.Connection = targetTable.Connection;
scd.CurrentRowWhere = "[_scdFrom] IS NOT NULL AND [_scdFrom] IS NULL";
scd.EnableInferredMember = targetTable.LateArriving;
scd.FailOnFixedAttributeChange = true;
scd.FailOnLookupFailure = false;
scd.IncomingRowChangeType = 1;
scd.InferredMemberIndicator = lateArrivingStatusColumn;
// TODO:
foreach (var column in targetTable.Columns)
{
if (column.IsAssignable && !column.IsAutoGenerated)
{
ScdColumnMappingType mappingType;
switch (column.ScdType)
{
case ScdType.Error:
mappingType = ScdColumnMappingType.FixedAttribute;
break;
case ScdType.Historical:
mappingType = ScdColumnMappingType.HistoricalAttribute;
break;
case ScdType.Key:
mappingType = ScdColumnMappingType.Key;
break;
case ScdType.Other:
mappingType = ScdColumnMappingType.Other;
break;
case ScdType.Update:
mappingType = ScdColumnMappingType.ChangingAttribute;
break;
default:
mappingType = ScdColumnMappingType.Other;
break;
}
scd.Mappings.Add(new AstScdTypeColumnMappingNode(scd)
{
MappingType = mappingType, QueryColumnName = column.Name
});
}
}
scd.Query = TableLowerer.EmitSelectAllStatement(targetTable);
if (outputPath != null)
{
scd.InputPath = new AstDataflowMappedInputPathNode(scd)
{
OutputPath = outputPath
};
}
workflowFragment.Add(scd);
// Late Arriving Path
if (targetTable.LateArriving)
{
BuildLateArrivingPath(targetTable, parentItem, workflowFragment, scd.InferredMemberPath);
}
// Change Path
BuildChangePath(targetTable, parentItem, workflowFragment, scd.ChangingAttributePath);
// Historical Path
var historicalOutput = BuildHistoricalSubpath(targetTable, parentItem, workflowFragment, scd.HistoricalAttributePath);
// Union Historical and New Paths
var insertUnionAll = new AstUnionAllNode(parentItem)
{
Name = Utility.NameCleanerAndUniqifier(targetTable.Name + "_InsertUnionAll")
};
insertUnionAll.InputPaths.Add(new AstDataflowMappedInputPathNode(insertUnionAll)
{
OutputPath = scd.NewPath
});
insertUnionAll.InputPaths.Add(new AstDataflowMappedInputPathNode(insertUnionAll)
{
OutputPath = historicalOutput
});
workflowFragment.Add(insertUnionAll);
// Insert Path
BuildInsertPath(targetTable, parentItem, workflowFragment, insertUnionAll.OutputPath);
return(workflowFragment);
}