//Create Data Sets and Pipelines - Deploy these in Azure Data Factory for a all structures in the source system. public static void DeployADFDataSetsAndPipelines() { List <string> tableNames = ADFOperations.ListFilteredTableNames(); foreach (string tableName in tableNames) { InitialParams.TableName = tableName; DeployADFDataSetsAndPipelines(CopyOnPremSQLToADLAType.All); DeployADFDataSetsAndPipelines(CopyOnPremSQLToADLAType.Distinct); DeployADFDataSetsAndPipelines(CopyOnPremSQLToADLAType.Transactional); } }
static void Main(string[] args) { try { argList = args; if (argList.First().Contains(":")) { LoadCommandLineArgs(); } else { SetCommandLinePropertyValues(); } ////Call Method: Create Data Sets, Pipelines for all structures qualified for criteria. if (InitialParams.DeployCriteria.Equals("search")) { string[] searchText = new string[3]; searchText[0] = InitialParams.SearchText01; searchText[1] = InitialParams.SearchText02; searchText[2] = InitialParams.SearchText03; ADFOperations.DeployADFDataSetsAndPipelines(searchText); } if (InitialParams.DeployCriteria.Equals("exact")) { ADFOperations.DeployADFDataSetsAndPipelines(InitialParams.OnPremiseADLAType); } if (InitialParams.DeployCriteria.Equals("delete")) { Console.WriteLine("Delete status of Input Data sets: Start "); //Console.WriteLine("Deleted Input Data set: End "); //Console.WriteLine("Delete status of Output Data sets: Start "); //ADFOperations.DeleteDatasets("SC_DSO_D_DSSnowdropLive"); Console.WriteLine("Deleted Output Data set: End "); } //ADFOperations.DeletePipelines("SC_DSO_H_PreProd_"); //ADFOperations.DeletePipelines("SC_DSI_H_PreProd_"); //ADFOperations.DeleteDatasets("SC_DSO_H_PreProd_"); //ADFOperations.DeleteDatasets("SC_DSI_H_PreProd_"); ////ADFOperations.DeleteDatasets("SC_DSI_H_PreProd_"); //ADFOperations.DeleteDatasets("SC_DSI_D_PreProd_"); Console.WriteLine("Completed the process of deploying ADF in azure"); Console.ReadLine(); } catch (Exception ex) { Console.WriteLine("Some exception occured: {0}", ex.Message); Console.ReadLine(); } }
//Create Data Sets and Pipelines - Deploy these in Azure Data Factory for a single Structure. public static void DeployADFDataSetsAndPipelines(CopyOnPremSQLToADLAType cpType) { //check if there is any data in the source structure. DateTime firstDateTimeRecordInTable; string tableName = InitialParams.TableName; if (System.String.IsNullOrEmpty(InitialParams.TableName)) { Console.WriteLine("source structure(SC table name) cannot be empty. Execution does not proceed further"); return; } //List the attribute, data type of each column List <DataElement> lstElements = ADFOperations.GenerateStructure(InitialParams.TableName); //if (System.String.IsNullOrEmpty(InitialParams.DataSourcePathInADLS)) //{ // InitialParams.DataSourcePathInADLS = Utils.GetdataSourceType(tableName); //} string dateTimeField = (String.IsNullOrEmpty(InitialParams.FilterDateTimeField) ? "recorddateutc" : InitialParams.FilterDateTimeField); folderPath = (String.IsNullOrEmpty(folderPath) ? ConfigurationSettings.AppSettings["folderPath"] : folderPath); firstDateTimeRecordInTable = ADFOperations.FetchFirstRowRecordDate(InitialParams.TableName, InitialParams.FilterDateTimeField); if (firstDateTimeRecordInTable <= DateTime.Now.Subtract(TimeSpan.FromHours(1))) { //re: INPUT DATASET - Prepare the SQL query required for pipeline to execute on Source System firstDateTimeRecordInTable = firstDateTimeRecordInTable.AddHours(3); string sqlQuery = ADFOperations.GenerateADFPipelineSQLQuery(lstElements, dateTimeField, false, cpType); string InOutDataSetNameRef = InitialParams.TablePathInADLS + "_" + cpType.ToString(); Console.WriteLine("Deploying data sets and pipelines for Headers"); string inDataSetName = String.Format("SC-{2}_DSI_H_{0}_{1}", InitialParams.Environment, InOutDataSetNameRef, InitialParams.TempCompPrefix); string outDataSetName = String.Format("SC-{2}_DSO_H_{0}_{1}", InitialParams.Environment, InOutDataSetNameRef, InitialParams.TempCompPrefix); string pipelineName = String.Format("SC-{2}_PL01_H_{0}_{1}", InitialParams.Environment, InOutDataSetNameRef, InitialParams.TempCompPrefix); string activityName = String.Format("Act-{2}_H_{0}_{1}", InitialParams.Environment, InOutDataSetNameRef, InitialParams.TempCompPrefix); string fileName = "Header_" + InOutDataSetNameRef; string folderpath = String.Format("{0}/{5}/DL-{1}/{4}/{2}/{3}", InitialParams.FolderPath, InitialParams.DataSourcePathInADLS, InitialParams.TablePathInADLS, InOutDataSetNameRef, InitialParams.TempCompPrefix, InitialParams.TempPathDeviation); DeployDatasetAndPipelines(pipelineName, activityName, inDataSetName, outDataSetName, tableName, lstElements, fileName, folderpath, sqlQuery, firstDateTimeRecordInTable, false, cpType); Console.WriteLine("Deployed data sets and pipelines for headers"); //re: OUTPUT DATASET - Prepare the SQL query required for pipeline to execute on Source System sqlQuery = ADFOperations.GenerateADFPipelineSQLQuery(lstElements, dateTimeField, true, cpType); Console.WriteLine("Deploying data sets and pipelines for data"); inDataSetName = String.Format("SC-{2}_DSI_D_{0}_{1}", InitialParams.Environment, InOutDataSetNameRef, InitialParams.TempCompPrefix); outDataSetName = String.Format("SC-{2}_DSO_D_{0}_{1}", InitialParams.Environment, InOutDataSetNameRef, InitialParams.TempCompPrefix); pipelineName = String.Format("SC-{2}_PL01_D_{0}_{1}", InitialParams.Environment, InOutDataSetNameRef, InitialParams.TempCompPrefix); activityName = String.Format("Act-{2}_D_{0}_{1}", InitialParams.Environment, InOutDataSetNameRef, InitialParams.TempCompPrefix); if (cpType.Equals(CopyOnPremSQLToADLAType.Distinct) || cpType.Equals(CopyOnPremSQLToADLAType.All) || cpType.Equals(CopyOnPremSQLToADLAType.Flattened) || cpType.Equals(CopyOnPremSQLToADLAType.LastIteration)) { fileName = "Data_" + InOutDataSetNameRef; } else { fileName = "Data_" + InOutDataSetNameRef + "-{year}-{month}-{day}-{hour}-{minute}"; } DeployDatasetAndPipelines(pipelineName, activityName, inDataSetName, outDataSetName, tableName, lstElements, fileName, folderpath, sqlQuery, firstDateTimeRecordInTable, true, cpType); Console.WriteLine("Deployed data sets and pipelines for data"); } else { Console.WriteLine(" Empty record date UTC for table : " + tableName); } }