public static HubTableMetadata GenerateFile(string sqlStatement, List <string> sqlStatements,
string outputFilePath, List <CsvDataSource> records)
{
var tableName = DDLParser.GetCreateDdlStatementTableName(sqlStatement);
tableName = tableName.ToLowerInvariant();
var hubTableMetadata = new HubTableMetadata();
try
{
Logger.LogInfo("Generating hub file for " + tableName);
hubTableMetadata.TableName = tableName;
hubTableMetadata.Columns = DDLParser.GetDdlStatementColumns(sqlStatement);
hubTableMetadata.PrimaryKeys = DDLParser.GetPrimaryKey(sqlStatements, tableName, records);
if (hubTableMetadata.PrimaryKeys.Contains(Constants.NotFoundString) && hubTableMetadata.PrimaryKeys.Count == 1)
{
hubTableMetadata.SrcPk = Constants.NotFoundString;
}
else
{
hubTableMetadata.SrcPk = hubTableMetadata.PrimaryKeys.Single(e => e.Contains("_HK", StringComparison.OrdinalIgnoreCase));
}
hubTableMetadata.SrcLdts = Constants.LoadTimestamp;
hubTableMetadata.SrcSource = Constants.RecordSource;
hubTableMetadata.SrcNk = new List <string>();
hubTableMetadata.Tags = CsvParser.GetTags(records, tableName);
foreach (var column in hubTableMetadata.Columns)
{
if (
hubTableMetadata.PrimaryKeys.Any(s => s.Equals(column.Name, StringComparison.OrdinalIgnoreCase)) ||
string.Equals(column.Name, hubTableMetadata.SrcLdts, StringComparison.OrdinalIgnoreCase) ||
string.Equals(column.Name, hubTableMetadata.SrcSource, StringComparison.OrdinalIgnoreCase))
{
}
else
{
hubTableMetadata.SrcNk.Add(column.Name);
}
}
hubTableMetadata.SourceModel = CsvParser.GetSourceModel(records, tableName);
outputFilePath += "hubs";
Utility.CreateDirectoryIfDoesNotExists(outputFilePath);
var hubFileTemplate = new HubFileTemplate(hubTableMetadata);
var content = hubFileTemplate.TransformText();
var pathStr = $"{outputFilePath}\\{hubTableMetadata.TableName}.sql";
File.WriteAllText(pathStr, content);
Logger.LogInfo("Generated hub file for " + tableName);
}
catch (Exception e)
{
Logger.LogError(e, Utility.ErrorGeneratingFileForTable("HUB", tableName, e.Message), "{@HubTableMetadata}", hubTableMetadata);
}
return(hubTableMetadata);
}
private static void GenerateModels(string ddlFilePath, string csvFilePath, string models, string outputFilePath)
{
var sqlStatements = DDLParser.BuildDdlStatementsCollection(ddlFilePath);
var records = CsvParser.ParseCsv(csvFilePath, models);
var fileNameArr = models.Split(',');
foreach (var sqlStatement in sqlStatements.Where(sqlStatement => !string.IsNullOrWhiteSpace(sqlStatement)))
{
if (Array.Exists(fileNameArr, element => string.Equals(element, Constants.HubFileName, StringComparison.OrdinalIgnoreCase) ||
string.Equals(element, "*", StringComparison.OrdinalIgnoreCase)))
{
if (sqlStatement.Contains("CREATE TABLE HUB", StringComparison.OrdinalIgnoreCase))
{
var hubTableMetadata = HubFileGenerator.GenerateFile(sqlStatement, sqlStatements, outputFilePath, records);
YamlFileGenerator.GenerateFile(sqlStatement, outputFilePath, hubTableMetadata.TableName, records);
DocFileGenerator.GenerateFile(outputFilePath, hubTableMetadata.TableName);
}
}
if (Array.Exists(fileNameArr, element => string.Equals(element, Constants.LnkFileName, StringComparison.OrdinalIgnoreCase) ||
string.Equals(element, "*", StringComparison.OrdinalIgnoreCase)))
{
if (sqlStatement.Contains("CREATE TABLE LNK", StringComparison.OrdinalIgnoreCase))
{
var linkTableMetadata = LinkFileGenerator.GenerateFile(sqlStatement, sqlStatements, outputFilePath, records);
YamlFileGenerator.GenerateFile(sqlStatement, outputFilePath, linkTableMetadata.TableName, records);
DocFileGenerator.GenerateFile(outputFilePath, linkTableMetadata.TableName);
}
}
if (Array.Exists(fileNameArr, element => string.Equals(element, Constants.SatFileName, StringComparison.OrdinalIgnoreCase) ||
string.Equals(element, "*", StringComparison.OrdinalIgnoreCase)))
{
if (sqlStatement.Contains("CREATE TABLE SAT", StringComparison.OrdinalIgnoreCase) || sqlStatement.Contains("CREATE TABLE MAS", StringComparison.OrdinalIgnoreCase))
{
var satTableMetadata = SatFileGenerator.GenerateFile(sqlStatement, sqlStatements, outputFilePath, records);
YamlFileGenerator.GenerateFile(sqlStatement, outputFilePath, satTableMetadata.TableName, records);
DocFileGenerator.GenerateFile(outputFilePath, satTableMetadata.TableName);
//GenerateStgFile(outputFilePath, fileNameArr, records, satTableMetadata.TableName, satTableMetadata.PrimaryKeys);
}
}
}
if (records != null && Array.Exists(fileNameArr,
element => string.Equals(element, "stg", StringComparison.OrdinalIgnoreCase) ||
string.Equals(element, "*", StringComparison.OrdinalIgnoreCase)))
{
StgFileGenerator.GenerateFile(records, outputFilePath);
}
}
public static void GenerateFile(string sqlStatement, string outputFilePath, string tableName, List <CsvDataSource> csvDataSource)
{
var yamlFileMetadata = new YamlFileMetadata();
yamlFileMetadata.TableDefinition = "";
yamlFileMetadata.TableName = tableName;
List <CsvDataSource> tableRecords = null;
yamlFileMetadata.ColumnsWithNotNullTest = new List <string>();
var nullOptionsPresent = false;
try
{
if (csvDataSource != null)
{
tableRecords = csvDataSource.Where(e => e.TableName.Equals(tableName, StringComparison.OrdinalIgnoreCase)).ToList();
foreach (var record in tableRecords)
{
if (record.NullOption != null)
{
nullOptionsPresent = true;
break;
}
}
if (nullOptionsPresent)
{
var NotNullRecords = tableRecords.Where(e => e.NullOption.Equals("Not Null", StringComparison.OrdinalIgnoreCase)).ToList();
yamlFileMetadata.ColumnsWithNotNullTest = NotNullRecords.Select(e => e.ColumnName).Distinct().ToList();
}
}
var columns = DDLParser.GetDdlStatementColumns(sqlStatement);
if (tableRecords == null || !tableRecords.Any())
{
List <LabelValuePair> columnsDetails = new List <LabelValuePair>();
foreach (var column in columns)
{
LabelValuePair columnDetails = new LabelValuePair();
columnDetails.Label = column.Name;
columnDetails.Value = "";
columnsDetails.Add(columnDetails);
}
yamlFileMetadata.ColumnDefinition = columnsDetails;
}
else
{
yamlFileMetadata.TableDefinition = GetTableDefinition(tableRecords);
yamlFileMetadata.ColumnDefinition = GetColumnDefinition(tableRecords, columns);
}
yamlFileMetadata.TableDefinition = Regex.Replace(yamlFileMetadata.TableDefinition, @"\s+", " ");
tableName = tableName.ToLowerInvariant();
Logger.LogInfo("Generating yml file for " + tableName);
if (tableName.Contains(Constants.HubFileName, StringComparison.OrdinalIgnoreCase))
{
outputFilePath += "hubs";
}
if (tableName.Contains(Constants.LnkFileName, StringComparison.OrdinalIgnoreCase))
{
outputFilePath += "links";
}
if (tableName.Contains(Constants.SatFileName, StringComparison.OrdinalIgnoreCase) || tableName.Contains(Constants.MasFileName, StringComparison.OrdinalIgnoreCase))
{
if (tableName.Contains(Constants.SatBrFileName, StringComparison.OrdinalIgnoreCase))
{
outputFilePath += "satellitebusinessrules";
}
else
{
outputFilePath += "satellites";
}
}
Utility.CreateDirectoryIfDoesNotExists(outputFilePath);
var yamlFileTemplate = new YamlFileTemplate(yamlFileMetadata);
var content = yamlFileTemplate.TransformText();
var pathStr = $"{outputFilePath}\\{tableName}.yml";
File.WriteAllText(pathStr, content);
Logger.LogInfo("Generated yml file for " + tableName);
}
catch (Exception e)
{
Logger.LogError(e, Utility.ErrorGeneratingFileForTable("yml", tableName, e.Message));
}
}
public static SatTableMetadata GenerateFile(string sqlStatement, List <string> sqlStatements,
string outputFilePath, List <CsvDataSource> records)
{
var tableName = DDLParser.GetCreateDdlStatementTableName(sqlStatement);
tableName = tableName.ToLowerInvariant();
var satTableMetadata = new SatTableMetadata();
try
{
Logger.LogInfo("Generating sat file for " + tableName);
satTableMetadata.TableName = tableName;
satTableMetadata.Columns = DDLParser.GetDdlStatementColumns(sqlStatement);
satTableMetadata.PrimaryKeys = DDLParser.GetPrimaryKey(sqlStatements, tableName, records);
if (satTableMetadata.PrimaryKeys.Contains(Constants.NotFoundString) && satTableMetadata.PrimaryKeys.Count == 1)
{
satTableMetadata.SrcPk = Constants.NotFoundString;
}
else
{
satTableMetadata.SrcPk = satTableMetadata.PrimaryKeys.Single(e => e.Contains("_HK", StringComparison.OrdinalIgnoreCase));
}
satTableMetadata.SrcHashDiff = Constants.SrcHashDiff;
satTableMetadata.SrcEff = Constants.SrcEff;
satTableMetadata.SrcLdts = Constants.LoadTimestamp;
satTableMetadata.SrcSource = Constants.RecordSource;
satTableMetadata.SrcPayload = new List <string>();
satTableMetadata.IsFIleTypeMAS = false;
satTableMetadata.CompositeKeysPresent = false;
if (satTableMetadata.PrimaryKeys.Count() > 1)
{
satTableMetadata.CompositeKeysPresent = true;
satTableMetadata.Compositekeys = satTableMetadata.PrimaryKeys.Where(e => !e.Contains("_HK", StringComparison.OrdinalIgnoreCase)).ToList();
}
satTableMetadata.Tags = CsvParser.GetTags(records, tableName);
foreach (var column in satTableMetadata.Columns)
{
if (
string.Equals(column.Name, satTableMetadata.SrcPk, StringComparison.OrdinalIgnoreCase) ||
//satTableMetadata.PrimaryKeys.Any(s => s.Equals(column.Name, StringComparison.OrdinalIgnoreCase)) ||
string.Equals(column.Name, satTableMetadata.SrcHashDiff, StringComparison.OrdinalIgnoreCase) ||
string.Equals(column.Name, satTableMetadata.SrcEff, StringComparison.OrdinalIgnoreCase) ||
string.Equals(column.Name, satTableMetadata.SrcLdts, StringComparison.OrdinalIgnoreCase) ||
string.Equals(column.Name, satTableMetadata.SrcSource, StringComparison.OrdinalIgnoreCase))
{
}
else
{
satTableMetadata.SrcPayload.Add(column.Name);
}
}
if (tableName.Contains("MAS", StringComparison.OrdinalIgnoreCase))
{
satTableMetadata.SourceModel = new List <string> {
Constants.NotFoundString
};
satTableMetadata.IsFIleTypeMAS = true;
}
else
{
satTableMetadata.SourceModel = CsvParser.GetSourceModel(records, tableName);
}
outputFilePath += "satellites";
Utility.CreateDirectoryIfDoesNotExists(outputFilePath);
var satFileTemplate = new SatFileTemplate(satTableMetadata);
var content = satFileTemplate.TransformText();
if (satTableMetadata.TableName.StartsWith(Constants.SatBrFileName, StringComparison.OrdinalIgnoreCase))
{
outputFilePath = outputFilePath.Remove(outputFilePath.Length - 1, 1) + "businessrules";
}
Utility.CreateDirectoryIfDoesNotExists(outputFilePath);
var pathStr = $"{outputFilePath}\\{satTableMetadata.TableName}.sql";
File.WriteAllText(pathStr, content);
Logger.LogInfo("Generated sat file for " + tableName);
}
catch (Exception e)
{
Logger.LogError(e, Utility.ErrorGeneratingFileForTable("SAT", tableName, e.Message), "{@SatTableMetadata}", satTableMetadata);
}
return(satTableMetadata);
}
