private static List <LabelListPair> GetDerivedColumns(List <CsvDataSource> tableRecords) { List <LabelListPair> derivedColumns = new List <LabelListPair>(); // Ignore rows containing _HK and Timestamp in the "columnname" column. tableRecords = tableRecords.Where(e => !e.ColumnName.Contains("_HK") && !e.ColumnName.Equals(Constants.LoadTimestamp) && !string.IsNullOrWhiteSpace(e.StageColumns)).ToList(); List <string> distinctColumnNameList = tableRecords.Select(e => e.ColumnName).Distinct().ToList(); foreach (var distinctColumnName in distinctColumnNameList) { LabelListPair derivedRecord = new LabelListPair(); // Getting values from column 'StageColumns' to be mapped with column names derivedRecord.Label = distinctColumnName; List <string> columnValuesList = new List <string>(); foreach (var record in tableRecords) { if (record.ColumnName.Equals(distinctColumnName)) { columnValuesList.Add(record.StageColumns); } } derivedRecord.Value = columnValuesList.Distinct().ToList(); derivedColumns.Add(derivedRecord); } foreach (var cols in derivedColumns) { if (cols.Label.Equals("RECORD_SOURCE")) { var recordSource = "!" + cols.Value[0].Replace("'", string.Empty); cols.Value = new List <string> { recordSource }; } if (cols.Label.Equals("EFFECTIVE_TIMESTAMP")) { cols.Value = new List <string> { "TO_TIMESTAMP(EFFECTIVEDATE)" }; } } return(derivedColumns); }
private static List <LabelListPair> GetHashedColumns(List <CsvDataSource> tableRecords) { //Hashed Columns will only contain rows which have _HK in the "columnname" column var hashedColumns = new List <LabelListPair>(); var hashedColumnRecords = tableRecords.Where(e => e.ColumnName.Contains("_HK")).ToList(); var distinctHashedColumnNames = hashedColumnRecords.Select(e => e.ColumnName).Distinct().ToList(); foreach (var distinctHashedColumnName in distinctHashedColumnNames) { var hashedColumnRecord = new LabelListPair { Label = distinctHashedColumnName }; var columnValuesList = new List <string>(); foreach (var record in hashedColumnRecords) { if (record.ColumnName.Equals(distinctHashedColumnName)) { //Getting hashed column values by breaking up the MD5 string if available if (record.StageColumns.Contains("MD5", StringComparison.OrdinalIgnoreCase)) { var pFrom = record.StageColumns.LastIndexOf("(", StringComparison.Ordinal) + 1; var pTo = record.StageColumns.IndexOf(")", StringComparison.Ordinal); columnValuesList.AddRange(record.StageColumns.Substring(pFrom, pTo - pFrom).Replace(" ", string.Empty).Split(",").ToList()); } } } var tableName = tableRecords[0].TableName; if (!columnValuesList.Any()) { columnValuesList.Add(Constants.NotFoundString); Logger.LogWarning($"Could not find MD5 values for the key: {distinctHashedColumnName} in SourceTransform column for table: {tableName}"); } hashedColumnRecord.Value = columnValuesList.Distinct().ToList(); hashedColumns.Add(hashedColumnRecord); } return(hashedColumns); }