public void TestOnPremDatasourceEncryption_LinkedServiceTypeParsing() { // Built-in linked service types Assert.True(DataFactoryClient.GetLinkedServiceType("OnPremisesSqlLinkedService") == LinkedServiceType.OnPremisesSqlLinkedService); Assert.True(DataFactoryClient.GetLinkedServiceType("OnPremisesFileSystemLinkedService") == LinkedServiceType.OnPremisesFileSystemLinkedService); Assert.True(DataFactoryClient.GetLinkedServiceType("OnPremisesOracleLinkedService") == LinkedServiceType.OnPremisesOracleLinkedService); Assert.True(DataFactoryClient.GetLinkedServiceType("OnPremisesOdbcLinkedService") == LinkedServiceType.OnPremisesOdbcLinkedService); Assert.True(DataFactoryClient.GetLinkedServiceType("OnPremisesPostgreSqlLinkedService") == LinkedServiceType.OnPremisesPostgreSqlLinkedService); Assert.True(DataFactoryClient.GetLinkedServiceType("OnPremisesTeradataLinkedService") == LinkedServiceType.OnPremisesTeradataLinkedService); Assert.True(DataFactoryClient.GetLinkedServiceType("OnPremisesMySQLLinkedService") == LinkedServiceType.OnPremisesMySQLLinkedService); Assert.True(DataFactoryClient.GetLinkedServiceType("OnPremisesDB2LinkedService") == LinkedServiceType.OnPremisesDB2LinkedService); Assert.True(DataFactoryClient.GetLinkedServiceType("OnPremisesSybaseLinkedService") == LinkedServiceType.OnPremisesSybaseLinkedService); // Generic linked service types should be converted to Unknown type Assert.True(DataFactoryClient.GetLinkedServiceType("HdfsLinkedService") == LinkedServiceType.Unknown); }
public static JObject CheckPipelineStatusMethod(string subscriptionId, string resourceGroup, string factoryName, string pipelineName, string runId, Logging logging) { #region CreatePipelineRun //Create a data factory management client logging.LogInformation("Creating ADF connectivity client."); string outputString = string.Empty; using (var client = DataFactoryClient.CreateDataFactoryClient(subscriptionId)) { //Get pipeline status with provided run id PipelineRun pipelineRun; pipelineRun = client.PipelineRuns.Get(resourceGroup, factoryName, runId); logging.LogInformation("Checking ADF pipeline status."); //Create simple status for Data Factory Until comparison checks string simpleStatus; if (pipelineRun.Status == "InProgress") { simpleStatus = "Running"; } else { simpleStatus = "Done"; } logging.LogInformation("ADF pipeline status: " + pipelineRun.Status); //Final return detail outputString = "{ \"PipelineName\": \"" + pipelineName + "\", \"RunId\": \"" + pipelineRun.RunId + "\", \"SimpleStatus\": \"" + simpleStatus + "\", \"Status\": \"" + pipelineRun.Status + "\" }"; } #endregion JObject outputJson = JObject.Parse(outputString); return(outputJson); }
public static async Task <IActionResult> Run( [HttpTrigger(AuthorizationLevel.Function, "get", "post", Route = null)] HttpRequest req, ILogger log) { log.LogInformation("CheckPipelineStatus Function triggered by HTTP request."); #region ParseRequestBody log.LogInformation("Parsing body from request."); string requestBody = await new StreamReader(req.Body).ReadToEndAsync(); dynamic data = JsonConvert.DeserializeObject(requestBody); string outputString = string.Empty; string tenantId = data?.tenantId; string applicationId = data?.applicationId; string authenticationKey = data?.authenticationKey; string subscriptionId = data?.subscriptionId; string resourceGroup = data?.resourceGroup; string factoryName = data?.factoryName; string pipelineName = data?.pipelineName; string runId = data?.runId; //Check body for values if ( tenantId == null || applicationId == null || authenticationKey == null || subscriptionId == null || resourceGroup == null || factoryName == null || pipelineName == null || runId == null ) { log.LogInformation("Invalid body."); return(new BadRequestObjectResult("Invalid request body, value(s) missing.")); } #endregion #region ResolveKeyVaultValues log.LogInformation(RequestHelper.CheckGuid(applicationId).ToString()); if (!RequestHelper.CheckGuid(applicationId) && RequestHelper.CheckUri(applicationId)) { log.LogInformation("Getting applicationId from Key Vault"); applicationId = KeyVaultClient.GetSecretFromUri(applicationId); } if (RequestHelper.CheckUri(authenticationKey)) { log.LogInformation("Getting authenticationKey from Key Vault"); authenticationKey = KeyVaultClient.GetSecretFromUri(authenticationKey); } #endregion #region GetPipelineStatus //Create a data factory management client log.LogInformation("Creating ADF connectivity client."); using (var adfClient = DataFactoryClient.CreateDataFactoryClient(tenantId, applicationId, authenticationKey, subscriptionId)) { //Get pipeline status with provided run id PipelineRun pipelineRun; pipelineRun = adfClient.PipelineRuns.Get(resourceGroup, factoryName, runId); log.LogInformation("Checking ADF pipeline status."); //Create simple status for Data Factory Until comparison checks string simpleStatus; if (pipelineRun.Status == "InProgress") { simpleStatus = "Running"; } else { simpleStatus = "Done"; } log.LogInformation("ADF pipeline status: " + pipelineRun.Status); //Final return detail outputString = "{ \"PipelineName\": \"" + pipelineName + "\", \"RunId\": \"" + pipelineRun.RunId + "\", \"SimpleStatus\": \"" + simpleStatus + "\", \"Status\": \"" + pipelineRun.Status + "\" }"; } #endregion JObject outputJson = JObject.Parse(outputString); log.LogInformation("CheckPipelineStatus Function complete."); return(new OkObjectResult(outputJson)); }
public static void QueryPipelineRuns(string subscriptionId, string resourceGroup, string factoryName, string _rungroupid, DateTime startDT, DateTime endDT, Logging logging) { #region QueryPipelineRuns logging.LogInformation("Query ADF Pipeline Runs."); string outputString = string.Empty; DataTable dt = new DataTable(); dt.Columns.Add(new DataColumn("ExecutionUid", typeof(Guid))); dt.Columns.Add(new DataColumn("TaskInstanceId", typeof(Int64))); dt.Columns.Add(new DataColumn("TaskMasterId", typeof(Int64))); //dt.Columns.Add(new DataColumn("AdditionalProperties", typeof(String))); dt.Columns.Add(new DataColumn("DurationInMs", typeof(Int64))); //dt.Columns.Add(new DataColumn("InvokedBy", typeof(String))); dt.Columns.Add(new DataColumn("IsLastest", typeof(Boolean))); dt.Columns.Add(new DataColumn("LastUpdated", typeof(DateTime))); //dt.Columns.Add(new DataColumn("Message", typeof(String))); //dt.Columns.Add(new DataColumn("Parameters", typeof(String))); dt.Columns.Add(new DataColumn("RunId", typeof(Guid))); //dt.Columns.Add(new DataColumn("RunGroupId", typeof(Guid))); dt.Columns.Add(new DataColumn("PipelineName", typeof(String))); dt.Columns.Add(new DataColumn("RunStart", typeof(DateTime))); dt.Columns.Add(new DataColumn("RunEnd", typeof(DateTime))); dt.Columns.Add(new DataColumn("RunDimensions", typeof(String))); dt.Columns.Add(new DataColumn("Status", typeof(String))); DataTable ActivityDt = new DataTable(); ActivityDt.Columns.Add(new DataColumn("ActivityName", typeof(String))); ActivityDt.Columns.Add(new DataColumn("RunId", typeof(Guid))); ActivityDt.Columns.Add(new DataColumn("ActivityRunStart", typeof(DateTime))); ActivityDt.Columns.Add(new DataColumn("ActivityRunEnd", typeof(DateTime))); ActivityDt.Columns.Add(new DataColumn("ActivityRunId", typeof(Guid))); ActivityDt.Columns.Add(new DataColumn("ActivityType", typeof(String))); //dt.Columns.Add(new DataColumn("AdditionalProperties", typeof(String))); ActivityDt.Columns.Add(new DataColumn("DurationInMs", typeof(Int64))); //dt.Columns.Add(new DataColumn("Error", typeof(String))); //dt.Columns.Add(new DataColumn("Input", typeof(String))); //dt.Columns.Add(new DataColumn("LinkedServiceName", typeof(String))); ActivityDt.Columns.Add(new DataColumn("OutPut", typeof(String))); ActivityDt.Columns.Add(new DataColumn("PipelineName", typeof(String))); ActivityDt.Columns.Add(new DataColumn("PipelineRunId", typeof(String))); ActivityDt.Columns.Add(new DataColumn("Status", typeof(String))); using (var client = DataFactoryClient.CreateDataFactoryClient(subscriptionId)) { //Get pipeline status with provided run id PipelineRunsQueryResponse pipelineRunsQueryResponse; RunFilterParameters filterParameterActivityRuns = new RunFilterParameters(); filterParameterActivityRuns.LastUpdatedAfter = startDT; filterParameterActivityRuns.LastUpdatedBefore = endDT.AddHours(+2); RunFilterParameters filterParameter = new RunFilterParameters(); filterParameter.LastUpdatedAfter = startDT; filterParameter.LastUpdatedBefore = endDT; IList <string> rungroupid = new List <string> { _rungroupid }; IList <RunQueryFilter> filter = new List <RunQueryFilter>(); filter.Add(new RunQueryFilter { Operand = RunQueryFilterOperand.RunGroupId, OperatorProperty = RunQueryFilterOperator.Equals, Values = rungroupid }); filterParameter.Filters = filter; logging.LogInformation(String.Format("API PipelineRuns.QueryByFactory Start")); pipelineRunsQueryResponse = client.PipelineRuns.QueryByFactory(resourceGroup, factoryName, filterParameter); logging.LogInformation(String.Format("API PipelineRuns.QueryByFactory End")); var enumerator = pipelineRunsQueryResponse.Value.GetEnumerator(); PipelineRun pipelineRuns; string runId = String.Empty; int item = 0; while (true) { for (bool hasMoreRuns = enumerator.MoveNext(); hasMoreRuns;) { pipelineRuns = enumerator.Current; hasMoreRuns = enumerator.MoveNext(); runId = pipelineRuns.RunId; item += 1; logging.LogInformation(String.Format("PipelineRuns.QueryByFactory RunId {0} Current Item {1} of {2}", runId, item, pipelineRunsQueryResponse.Value.Count)); DataRow dr = dt.NewRow(); string _param = string.Empty; foreach (var element in pipelineRuns.Parameters) { _param = element.Value; break; } dr["ExecutionUid"] = Shared.JsonHelpers.GetStringValueFromJSON(logging, "ExecutionUid", JObject.Parse(_param), null, true); dr["TaskInstanceId"] = Shared.JsonHelpers.GetStringValueFromJSON(logging, "TaskInstanceId", JObject.Parse(_param), null, true); dr["TaskMasterId"] = Shared.JsonHelpers.GetStringValueFromJSON(logging, "TaskMasterId", JObject.Parse(_param), null, true); //dr["AdditionalProperties"] = pipelineRuns.AdditionalProperties ?? (object)DBNull.Value; dr["DurationInMs"] = pipelineRuns.DurationInMs ?? (object)DBNull.Value; //dr["InvokedBy"] = pipelineRuns.InvokedBy ?? (object)DBNull.Value; dr["IsLastest"] = pipelineRuns.IsLatest ?? (object)DBNull.Value; dr["LastUpdated"] = pipelineRuns.LastUpdated ?? (object)DBNull.Value; //dr["Message"] = pipelineRuns.Message ?? (object)DBNull.Value; //dr["Parameters"] = _param; dr["RunId"] = pipelineRuns.RunId; //dr["RunGroupId"] = pipelineRuns.RunGroupId ?? (object)DBNull.Value; dr["PipelineName"] = pipelineRuns.PipelineName ?? (object)DBNull.Value; dr["RunStart"] = pipelineRuns.RunStart ?? (object)DBNull.Value; dr["RunEnd"] = pipelineRuns.RunEnd ?? (object)DBNull.Value; dr["RunDimensions"] = pipelineRuns.PipelineName ?? (object)DBNull.Value; dr["Status"] = pipelineRuns.Status ?? (object)DBNull.Value; dt.Rows.Add(dr); QueryActivityRuns(subscriptionId, resourceGroup, factoryName, runId, runId, filterParameterActivityRuns, logging, ref ActivityDt); } if (pipelineRunsQueryResponse.ContinuationToken == null) { break; } filterParameter.ContinuationToken = pipelineRunsQueryResponse.ContinuationToken; pipelineRunsQueryResponse = client.PipelineRuns.QueryByFactory(resourceGroup, factoryName, filterParameter); enumerator = pipelineRunsQueryResponse.Value.GetEnumerator(); item = 0; } } if (ActivityDt.Rows.Count > 0) { string TempTableName = "#Temp_ADFActivities_" + Guid.NewGuid().ToString(); TaskMetaDataDatabase TMD = new TaskMetaDataDatabase(); TMD.AutoBulkInsertAndMerge(ActivityDt, TempTableName, "ADFActivity"); } if (dt.Rows.Count > 0) { string TempTableName = "#Temp_ADFPipelineRun_" + Guid.NewGuid().ToString(); TaskMetaDataDatabase TMD = new TaskMetaDataDatabase(); TMD.AutoBulkInsertAndMerge(dt, TempTableName, "ADFPipelineRun"); } #endregion }
public static JObject ExecutePipelineMethod(string subscriptionId, string resourceGroup, string factoryName, string pipelineName, Dictionary <string, object> parameters, Logging logging) { #region CreatePipelineRun //Create a data factory management client logging.LogInformation("Creating ADF connectivity client."); string outputString = string.Empty; using (var client = DataFactoryClient.CreateDataFactoryClient(subscriptionId)) { //Run pipeline CreateRunResponse runResponse; PipelineRun pipelineRun; if (parameters.Count == 0) { logging.LogInformation("Called pipeline without parameters."); runResponse = client.Pipelines.CreateRunWithHttpMessagesAsync( resourceGroup, factoryName, pipelineName).Result.Body; } else { logging.LogInformation("Called pipeline with parameters."); logging.LogInformation("Number of parameters provided: " + parameters.Count); System.Threading.Thread.Sleep(1000); runResponse = client.Pipelines.CreateRunWithHttpMessagesAsync( resourceGroup, factoryName, pipelineName, parameters: parameters).Result.Body; runResponse = new CreateRunResponse(); runResponse.RunId = Guid.NewGuid().ToString(); } logging.LogInformation("Pipeline run ID: " + runResponse.RunId); //Wait and check for pipeline to start... logging.LogInformation("Checking ADF pipeline status."); //while (true) //{ // pipelineRun = client.PipelineRuns.Get( // resourceGroup, factoryName, runResponse.RunId); // logging.LogInformation("ADF pipeline status: " + pipelineRun.Status); //if (pipelineRun.Status == "Queued") // System.Threading.Thread.Sleep(15000); //else // break; //} //Final return detail outputString = "{ \"PipelineName\": \"" + pipelineName + "\", \"RunId\": \"" + runResponse.RunId + "\", \"Status\": \"" + "InProgress" + //pipelineRun.Status + "\" }"; } #endregion JObject outputJson = JObject.Parse(outputString); return(outputJson); }
public static void QueryActivityRuns(string subscriptionId, string resourceGroup, string factoryName, string runId, string parentRunId, RunFilterParameters filterParameterActivityRuns, Logging logging, ref DataTable dt) { #region QueryActivityRuns logging.LogInformation(String.Format("QueryActivityRuns - RunId {0}", runId)); string outputString = string.Empty; using (var client = DataFactoryClient.CreateDataFactoryClient(subscriptionId)) { //Get pipeline status with provided run id logging.LogInformation(String.Format("QueryActivityRuns - RunId {0} - API QueryByPipelineRun Start", runId)); ActivityRunsQueryResponse activityRunsQueryResponse = client.ActivityRuns.QueryByPipelineRun(resourceGroup, factoryName, runId, filterParameterActivityRuns); logging.LogInformation(String.Format("QueryActivityRuns - RunId {0} - API QueryByPipelineRun End", runId)); var enumeratorActivity = activityRunsQueryResponse.Value.GetEnumerator(); ActivityRun activityRun; for (bool hasMoreActivityRuns = enumeratorActivity.MoveNext(); hasMoreActivityRuns;) { activityRun = enumeratorActivity.Current; hasMoreActivityRuns = enumeratorActivity.MoveNext(); DataRow dr = dt.NewRow(); dr["ActivityName"] = activityRun.ActivityName ?? (object)DBNull.Value; dr["RunId"] = parentRunId; dr["ActivityRunStart"] = activityRun.ActivityRunStart ?? (object)DBNull.Value; dr["ActivityRunEnd"] = activityRun.ActivityRunEnd ?? (object)DBNull.Value; dr["ActivityRunId"] = activityRun.ActivityRunId ?? (object)DBNull.Value; dr["ActivityType"] = activityRun.ActivityType ?? (object)DBNull.Value; //dr["AdditionalProperties"] = activityRun.AdditionalProperties ?? (object)DBNull.Value; dr["DurationInMs"] = activityRun.DurationInMs ?? (object)DBNull.Value; //dr["Error"] = activityRun.Error ?? (object)DBNull.Value; //dr["Input"] = activityRun.Input ?? (object)DBNull.Value; //dr["LinkedServiceName"] = activityRun.LinkedServiceName ?? (object)DBNull.Value; dr["OutPut"] = activityRun.Output ?? (object)DBNull.Value; dr["PipelineName"] = activityRun.PipelineName ?? (object)DBNull.Value; dr["PipelineRunId"] = activityRun.PipelineRunId ?? (object)DBNull.Value; dr["Status"] = activityRun.Status ?? (object)DBNull.Value; dt.Rows.Add(dr); if (activityRun.ActivityType == "ExecutePipeline") { string _runId = Shared.JsonHelpers.GetStringValueFromJSON(logging, "pipelineRunId", (JObject)activityRun.Output, null, true); if (!String.IsNullOrEmpty(_runId)) { QueryActivityRuns(subscriptionId, resourceGroup, factoryName, _runId, parentRunId, filterParameterActivityRuns, logging, ref dt); } else { logging.LogInformation(String.Format("QueryActivityRuns - RunId Is null for {0} ", (JObject)activityRun.Output)); } } } } #endregion }
public static async Task <IActionResult> Run( [HttpTrigger(AuthorizationLevel.Function, "get", "post", Route = null)] HttpRequest req, ILogger log) { log.LogInformation("ValidatePipeline Function triggered by HTTP request."); #region ParseRequestBody log.LogInformation("Parsing body from request."); string requestBody = await new StreamReader(req.Body).ReadToEndAsync(); dynamic data = JsonConvert.DeserializeObject(requestBody); string outputString = string.Empty; string tenantId = data?.tenantId; string applicationId = data?.applicationId; string authenticationKey = data?.authenticationKey; string subscriptionId = data?.subscriptionId; string resourceGroup = data?.resourceGroup; string factoryName = data?.factoryName; string pipelineName = data?.pipelineName; //Check body for values if ( tenantId == null || applicationId == null || authenticationKey == null || subscriptionId == null || resourceGroup == null || factoryName == null || pipelineName == null ) { log.LogInformation("Invalid body."); return(new BadRequestObjectResult("Invalid request body, value(s) missing.")); } #endregion #region ResolveKeyVaultValues log.LogInformation(RequestHelper.CheckGuid(applicationId).ToString()); if (!RequestHelper.CheckGuid(applicationId) && RequestHelper.CheckUri(applicationId)) { log.LogInformation("Getting applicationId from Key Vault"); applicationId = KeyVaultClient.GetSecretFromUri(applicationId); } if (RequestHelper.CheckUri(authenticationKey)) { log.LogInformation("Getting authenticationKey from Key Vault"); authenticationKey = KeyVaultClient.GetSecretFromUri(authenticationKey); } #endregion #region ValidatePipeline //Create a data factory management client log.LogInformation("Creating ADF connectivity client."); using (var adfClient = DataFactoryClient.CreateDataFactoryClient(tenantId, applicationId, authenticationKey, subscriptionId)) { PipelineResource pipelineResource; try { pipelineResource = adfClient.Pipelines.Get(resourceGroup, factoryName, pipelineName); log.LogInformation(pipelineResource.Id.ToString()); log.LogInformation(pipelineResource.Name.ToString()); outputString = "{ \"PipelineExists\": \"" + true.ToString() + "\", \"PipelineName\": \"" + pipelineResource.Name.ToString() + "\", \"PipelineId\": \"" + pipelineResource.Id.ToString() + "\", \"PipelineType\": \"" + pipelineResource.Type.ToString() + "\", \"ActivityCount\": \"" + pipelineResource.Activities.Count.ToString() + "\" }"; } catch (Exception ex) { outputString = "{ \"PipelineExists\": \"" + false.ToString() + "\", \"ProvidedPipelineName\": \"" + pipelineName + "\" }"; log.LogInformation(ex.Message); } } #endregion JObject outputJson = JObject.Parse(outputString); log.LogInformation("CheckPipelineStatus Function complete."); //return false positive so PipelineExists result can be evaluated by Data Factory return(new OkObjectResult(outputJson)); }
public static async Task <IActionResult> Run( [HttpTrigger(AuthorizationLevel.Function, "post", Route = null)] HttpRequest req, ILogger log) { log.LogInformation("ExecutePipeline Function triggered by HTTP request."); #region ParseRequestBody log.LogInformation("Parsing body from request."); string requestBody = await new StreamReader(req.Body).ReadToEndAsync(); dynamic data = JsonConvert.DeserializeObject(requestBody); string tenantId = data?.tenantId; string applicationId = data?.applicationId; string authenticationKey = data?.authenticationKey; string subscriptionId = data?.subscriptionId; string resourceGroup = data?.resourceGroup; string factoryName = data?.factoryName; string pipelineName = data?.pipelineName; //Check body for values if ( tenantId == null || applicationId == null || authenticationKey == null || subscriptionId == null || resourceGroup == null || factoryName == null || pipelineName == null ) { log.LogInformation("Invalid body."); return(new BadRequestObjectResult("Invalid request body, value missing.")); } #endregion #region ResolveKeyVaultValues if (!RequestHelper.CheckGuid(applicationId) && RequestHelper.CheckUri(applicationId)) { log.LogInformation("Getting applicationId from Key Vault"); applicationId = KeyVaultClient.GetSecretFromUri(applicationId); } if (RequestHelper.CheckUri(authenticationKey)) { log.LogInformation("Getting authenticationKey from Key Vault"); authenticationKey = KeyVaultClient.GetSecretFromUri(authenticationKey); } #endregion #region CreatePipelineRun //Create a data factory management client log.LogInformation("Creating ADF connectivity client."); string outputString = string.Empty; using (var client = DataFactoryClient.CreateDataFactoryClient(tenantId, applicationId, authenticationKey, subscriptionId)) { //Run pipeline CreateRunResponse runResponse; PipelineRun pipelineRun; if (data?.pipelineParameters == null) { log.LogInformation("Called pipeline without parameters."); runResponse = client.Pipelines.CreateRunWithHttpMessagesAsync( resourceGroup, factoryName, pipelineName).Result.Body; } else { log.LogInformation("Called pipeline with parameters."); JObject jObj = JObject.Parse(requestBody); Dictionary <string, object> parameters = jObj["pipelineParameters"].ToObject <Dictionary <string, object> >(); log.LogInformation("Number of parameters provided: " + jObj.Count.ToString()); runResponse = client.Pipelines.CreateRunWithHttpMessagesAsync( resourceGroup, factoryName, pipelineName, parameters: parameters).Result.Body; } log.LogInformation("Pipeline run ID: " + runResponse.RunId); //Wait and check for pipeline to start... log.LogInformation("Checking ADF pipeline status."); while (true) { pipelineRun = client.PipelineRuns.Get( resourceGroup, factoryName, runResponse.RunId); log.LogInformation("ADF pipeline status: " + pipelineRun.Status); if (pipelineRun.Status == "Queued") { System.Threading.Thread.Sleep(15000); } else { break; } } //Final return detail outputString = "{ \"PipelineName\": \"" + pipelineName + "\", \"RunId\": \"" + pipelineRun.RunId + "\", \"Status\": \"" + pipelineRun.Status + "\" }"; } #endregion JObject outputJson = JObject.Parse(outputString); log.LogInformation("ExecutePipeline Function complete."); return(new OkObjectResult(outputJson)); }
public static async Task <IActionResult> Run( [HttpTrigger(AuthorizationLevel.Function, "get", "post", Route = null)] HttpRequest req, ILogger log) { log.LogInformation("GetActivityErrors Function triggered by HTTP request."); #region ParseRequestBody log.LogInformation("Parsing body from request."); string requestBody = await new StreamReader(req.Body).ReadToEndAsync(); dynamic inputData = JsonConvert.DeserializeObject(requestBody); string tenantId = inputData?.tenantId; string applicationId = inputData?.applicationId; string authenticationKey = inputData?.authenticationKey; string subscriptionId = inputData?.subscriptionId; string resourceGroup = inputData?.resourceGroup; string factoryName = inputData?.factoryName; string pipelineName = inputData?.pipelineName; string runId = inputData?.runId; //Check body for values if ( tenantId == null || applicationId == null || authenticationKey == null || subscriptionId == null || resourceGroup == null || factoryName == null || pipelineName == null || runId == null ) { log.LogInformation("Invalid body."); return(new BadRequestObjectResult("Invalid request body, value(s) missing.")); } #endregion //Query and output support variables int daysOfRuns = 7; //max duration for mandatory RunFilterParameters DateTime today = DateTime.Now; DateTime lastWeek = DateTime.Now.AddDays(-daysOfRuns); dynamic outputBlock = new JObject(); dynamic outputBlockInner; //Create a data factory management client log.LogInformation("Creating ADF connectivity client."); using (var client = DataFactoryClient.CreateDataFactoryClient(tenantId, applicationId, authenticationKey, subscriptionId)) { #region SetPipelineRunDetails //Get pipeline details PipelineRun pipelineRun; pipelineRun = client.PipelineRuns.Get(resourceGroup, factoryName, runId); log.LogInformation("Querying ADF pipeline for Activity Runs."); RunFilterParameters filterParams = new RunFilterParameters(lastWeek, today); ActivityRunsQueryResponse queryResponse = client.ActivityRuns.QueryByPipelineRun(resourceGroup, factoryName, runId, filterParams); //Create initial output content outputBlock.PipelineName = pipelineName; outputBlock.PipelineStatus = pipelineRun.Status; outputBlock.RunId = runId; outputBlock.ResponseCount = queryResponse.Value.Count; outputBlock.ResponseErrorCount = 0; outputBlock.Errors = new JArray(); JObject errorDetails; #endregion log.LogInformation("Pipeline status: " + pipelineRun.Status); log.LogInformation("Activities found in pipeline response: " + queryResponse.Value.Count.ToString()); #region GetActivityDetails //Loop over activities in pipeline run foreach (var activity in queryResponse.Value) { if (string.IsNullOrEmpty(activity.Error.ToString())) { continue; //just incase } //Parse error output to customise output outputBlockInner = JsonConvert.DeserializeObject(activity.Error.ToString()); string errorCode = outputBlockInner?.errorCode; string errorType = outputBlockInner?.failureType; string errorMessage = outputBlockInner?.message; //Get output details if (!string.IsNullOrEmpty(errorCode)) { log.LogInformation("Activity run id: " + activity.ActivityRunId); log.LogInformation("Activity name: " + activity.ActivityName); log.LogInformation("Activity type: " + activity.ActivityType); log.LogInformation("Error message: " + errorMessage); outputBlock.ResponseErrorCount += 1; //Construct custom error information block errorDetails = JObject.Parse("{ \"ActivityRunId\": \"" + activity.ActivityRunId + "\", \"ActivityName\": \"" + activity.ActivityName + "\", \"ActivityType\": \"" + activity.ActivityType + "\", \"ErrorCode\": \"" + errorCode + "\", \"ErrorType\": \"" + errorType + "\", \"ErrorMessage\": \"" + errorMessage + "\" }"); outputBlock.Errors.Add(errorDetails); } } #endregion } log.LogInformation("GetActivityErrors Function complete."); return(new OkObjectResult(outputBlock)); }