private static List <LabelListPair> GetDerivedColumns(List <CsvDataSource> tableRecords)
{
List <LabelListPair> derivedColumns = new List <LabelListPair>();
// Ignore rows containing _HK and Timestamp in the "columnname" column.
tableRecords = tableRecords.Where(e => !e.ColumnName.Contains("_HK") && !e.ColumnName.Equals(Constants.LoadTimestamp) && !string.IsNullOrWhiteSpace(e.StageColumns)).ToList();
List <string> distinctColumnNameList = tableRecords.Select(e => e.ColumnName).Distinct().ToList();
foreach (var distinctColumnName in distinctColumnNameList)
{
LabelListPair derivedRecord = new LabelListPair();
// Getting values from column 'StageColumns' to be mapped with column names
derivedRecord.Label = distinctColumnName;
List <string> columnValuesList = new List <string>();
foreach (var record in tableRecords)
{
if (record.ColumnName.Equals(distinctColumnName))
{
columnValuesList.Add(record.StageColumns);
}
}
derivedRecord.Value = columnValuesList.Distinct().ToList();
derivedColumns.Add(derivedRecord);
}
foreach (var cols in derivedColumns)
{
if (cols.Label.Equals("RECORD_SOURCE"))
{
var recordSource = "!" + cols.Value[0].Replace("'", string.Empty);
cols.Value = new List <string> {
recordSource
};
}
if (cols.Label.Equals("EFFECTIVE_TIMESTAMP"))
{
cols.Value = new List <string> {
"TO_TIMESTAMP(EFFECTIVEDATE)"
};
}
}
return(derivedColumns);
}
private static List <LabelListPair> GetHashedColumns(List <CsvDataSource> tableRecords)
{
//Hashed Columns will only contain rows which have _HK in the "columnname" column
var hashedColumns = new List <LabelListPair>();
var hashedColumnRecords = tableRecords.Where(e => e.ColumnName.Contains("_HK")).ToList();
var distinctHashedColumnNames = hashedColumnRecords.Select(e => e.ColumnName).Distinct().ToList();
foreach (var distinctHashedColumnName in distinctHashedColumnNames)
{
var hashedColumnRecord = new LabelListPair {
Label = distinctHashedColumnName
};
var columnValuesList = new List <string>();
foreach (var record in hashedColumnRecords)
{
if (record.ColumnName.Equals(distinctHashedColumnName))
{
//Getting hashed column values by breaking up the MD5 string if available
if (record.StageColumns.Contains("MD5", StringComparison.OrdinalIgnoreCase))
{
var pFrom = record.StageColumns.LastIndexOf("(", StringComparison.Ordinal) + 1;
var pTo = record.StageColumns.IndexOf(")", StringComparison.Ordinal);
columnValuesList.AddRange(record.StageColumns.Substring(pFrom, pTo - pFrom).Replace(" ", string.Empty).Split(",").ToList());
}
}
}
var tableName = tableRecords[0].TableName;
if (!columnValuesList.Any())
{
columnValuesList.Add(Constants.NotFoundString);
Logger.LogWarning($"Could not find MD5 values for the key: {distinctHashedColumnName} in SourceTransform column for table: {tableName}");
}
hashedColumnRecord.Value = columnValuesList.Distinct().ToList();
hashedColumns.Add(hashedColumnRecord);
}
return(hashedColumns);
}
