//Create Data Sets and Pipelines - Deploy these in Azure Data Factory for a all structures in the source system.
public static void DeployADFDataSetsAndPipelines()
{
List <string> tableNames = ADFOperations.ListFilteredTableNames();
foreach (string tableName in tableNames)
{
InitialParams.TableName = tableName;
DeployADFDataSetsAndPipelines(CopyOnPremSQLToADLAType.All);
DeployADFDataSetsAndPipelines(CopyOnPremSQLToADLAType.Distinct);
DeployADFDataSetsAndPipelines(CopyOnPremSQLToADLAType.Transactional);
}
}
static void Main(string[] args)
{
try
{
argList = args;
if (argList.First().Contains(":"))
{
LoadCommandLineArgs();
}
else
{
SetCommandLinePropertyValues();
}
////Call Method: Create Data Sets, Pipelines for all structures qualified for criteria.
if (InitialParams.DeployCriteria.Equals("search"))
{
string[] searchText = new string[3];
searchText[0] = InitialParams.SearchText01;
searchText[1] = InitialParams.SearchText02;
searchText[2] = InitialParams.SearchText03;
ADFOperations.DeployADFDataSetsAndPipelines(searchText);
}
if (InitialParams.DeployCriteria.Equals("exact"))
{
ADFOperations.DeployADFDataSetsAndPipelines(InitialParams.OnPremiseADLAType);
}
if (InitialParams.DeployCriteria.Equals("delete"))
{
Console.WriteLine("Delete status of Input Data sets: Start ");
//Console.WriteLine("Deleted Input Data set: End ");
//Console.WriteLine("Delete status of Output Data sets: Start ");
//ADFOperations.DeleteDatasets("SC_DSO_D_DSSnowdropLive");
Console.WriteLine("Deleted Output Data set: End ");
}
//ADFOperations.DeletePipelines("SC_DSO_H_PreProd_");
//ADFOperations.DeletePipelines("SC_DSI_H_PreProd_");
//ADFOperations.DeleteDatasets("SC_DSO_H_PreProd_");
//ADFOperations.DeleteDatasets("SC_DSI_H_PreProd_");
////ADFOperations.DeleteDatasets("SC_DSI_H_PreProd_");
//ADFOperations.DeleteDatasets("SC_DSI_D_PreProd_");
Console.WriteLine("Completed the process of deploying ADF in azure");
Console.ReadLine();
}
catch (Exception ex)
{
Console.WriteLine("Some exception occured: {0}", ex.Message);
Console.ReadLine();
}
}
//Create Data Sets and Pipelines - Deploy these in Azure Data Factory for a single Structure.
public static void DeployADFDataSetsAndPipelines(CopyOnPremSQLToADLAType cpType)
{
//check if there is any data in the source structure.
DateTime firstDateTimeRecordInTable;
string tableName = InitialParams.TableName;
if (System.String.IsNullOrEmpty(InitialParams.TableName))
{
Console.WriteLine("source structure(SC table name) cannot be empty. Execution does not proceed further");
return;
}
//List the attribute, data type of each column
List <DataElement> lstElements = ADFOperations.GenerateStructure(InitialParams.TableName);
//if (System.String.IsNullOrEmpty(InitialParams.DataSourcePathInADLS))
//{
// InitialParams.DataSourcePathInADLS = Utils.GetdataSourceType(tableName);
//}
string dateTimeField = (String.IsNullOrEmpty(InitialParams.FilterDateTimeField) ? "recorddateutc" : InitialParams.FilterDateTimeField);
folderPath = (String.IsNullOrEmpty(folderPath) ? ConfigurationSettings.AppSettings["folderPath"] : folderPath);
firstDateTimeRecordInTable = ADFOperations.FetchFirstRowRecordDate(InitialParams.TableName, InitialParams.FilterDateTimeField);
if (firstDateTimeRecordInTable <= DateTime.Now.Subtract(TimeSpan.FromHours(1)))
{
//re: INPUT DATASET - Prepare the SQL query required for pipeline to execute on Source System
firstDateTimeRecordInTable = firstDateTimeRecordInTable.AddHours(3);
string sqlQuery = ADFOperations.GenerateADFPipelineSQLQuery(lstElements, dateTimeField, false, cpType);
string InOutDataSetNameRef = InitialParams.TablePathInADLS + "_" + cpType.ToString();
Console.WriteLine("Deploying data sets and pipelines for Headers");
string inDataSetName = String.Format("SC-{2}_DSI_H_{0}_{1}", InitialParams.Environment, InOutDataSetNameRef, InitialParams.TempCompPrefix);
string outDataSetName = String.Format("SC-{2}_DSO_H_{0}_{1}", InitialParams.Environment, InOutDataSetNameRef, InitialParams.TempCompPrefix);
string pipelineName = String.Format("SC-{2}_PL01_H_{0}_{1}", InitialParams.Environment, InOutDataSetNameRef, InitialParams.TempCompPrefix);
string activityName = String.Format("Act-{2}_H_{0}_{1}", InitialParams.Environment, InOutDataSetNameRef, InitialParams.TempCompPrefix);
string fileName = "Header_" + InOutDataSetNameRef;
string folderpath = String.Format("{0}/{5}/DL-{1}/{4}/{2}/{3}", InitialParams.FolderPath,
InitialParams.DataSourcePathInADLS, InitialParams.TablePathInADLS, InOutDataSetNameRef, InitialParams.TempCompPrefix,
InitialParams.TempPathDeviation);
DeployDatasetAndPipelines(pipelineName, activityName, inDataSetName, outDataSetName, tableName,
lstElements, fileName, folderpath, sqlQuery, firstDateTimeRecordInTable, false, cpType);
Console.WriteLine("Deployed data sets and pipelines for headers");
//re: OUTPUT DATASET - Prepare the SQL query required for pipeline to execute on Source System
sqlQuery = ADFOperations.GenerateADFPipelineSQLQuery(lstElements, dateTimeField, true, cpType);
Console.WriteLine("Deploying data sets and pipelines for data");
inDataSetName = String.Format("SC-{2}_DSI_D_{0}_{1}", InitialParams.Environment, InOutDataSetNameRef, InitialParams.TempCompPrefix);
outDataSetName = String.Format("SC-{2}_DSO_D_{0}_{1}", InitialParams.Environment, InOutDataSetNameRef, InitialParams.TempCompPrefix);
pipelineName = String.Format("SC-{2}_PL01_D_{0}_{1}", InitialParams.Environment, InOutDataSetNameRef, InitialParams.TempCompPrefix);
activityName = String.Format("Act-{2}_D_{0}_{1}", InitialParams.Environment, InOutDataSetNameRef, InitialParams.TempCompPrefix);
if (cpType.Equals(CopyOnPremSQLToADLAType.Distinct) ||
cpType.Equals(CopyOnPremSQLToADLAType.All) ||
cpType.Equals(CopyOnPremSQLToADLAType.Flattened) ||
cpType.Equals(CopyOnPremSQLToADLAType.LastIteration))
{
fileName = "Data_" + InOutDataSetNameRef;
}
else
{
fileName = "Data_" + InOutDataSetNameRef + "-{year}-{month}-{day}-{hour}-{minute}";
}
DeployDatasetAndPipelines(pipelineName, activityName, inDataSetName, outDataSetName, tableName,
lstElements, fileName, folderpath, sqlQuery, firstDateTimeRecordInTable, true, cpType);
Console.WriteLine("Deployed data sets and pipelines for data");
}
else
{
Console.WriteLine(" Empty record date UTC for table : " + tableName);
}
}
