public void TestOnPremDatasourceEncryption_LinkedServiceTypeParsing()
{
// Built-in linked service types
Assert.True(DataFactoryClient.GetLinkedServiceType("OnPremisesSqlLinkedService") == LinkedServiceType.OnPremisesSqlLinkedService);
Assert.True(DataFactoryClient.GetLinkedServiceType("OnPremisesFileSystemLinkedService") == LinkedServiceType.OnPremisesFileSystemLinkedService);
Assert.True(DataFactoryClient.GetLinkedServiceType("OnPremisesOracleLinkedService") == LinkedServiceType.OnPremisesOracleLinkedService);
Assert.True(DataFactoryClient.GetLinkedServiceType("OnPremisesOdbcLinkedService") == LinkedServiceType.OnPremisesOdbcLinkedService);
Assert.True(DataFactoryClient.GetLinkedServiceType("OnPremisesPostgreSqlLinkedService") == LinkedServiceType.OnPremisesPostgreSqlLinkedService);
Assert.True(DataFactoryClient.GetLinkedServiceType("OnPremisesTeradataLinkedService") == LinkedServiceType.OnPremisesTeradataLinkedService);
Assert.True(DataFactoryClient.GetLinkedServiceType("OnPremisesMySQLLinkedService") == LinkedServiceType.OnPremisesMySQLLinkedService);
Assert.True(DataFactoryClient.GetLinkedServiceType("OnPremisesDB2LinkedService") == LinkedServiceType.OnPremisesDB2LinkedService);
Assert.True(DataFactoryClient.GetLinkedServiceType("OnPremisesSybaseLinkedService") == LinkedServiceType.OnPremisesSybaseLinkedService);
// Generic linked service types should be converted to Unknown type
Assert.True(DataFactoryClient.GetLinkedServiceType("HdfsLinkedService") == LinkedServiceType.Unknown);
}
public static JObject CheckPipelineStatusMethod(string subscriptionId, string resourceGroup, string factoryName, string pipelineName, string runId, Logging logging)
{
#region CreatePipelineRun
//Create a data factory management client
logging.LogInformation("Creating ADF connectivity client.");
string outputString = string.Empty;
using (var client = DataFactoryClient.CreateDataFactoryClient(subscriptionId))
{
//Get pipeline status with provided run id
PipelineRun pipelineRun;
pipelineRun = client.PipelineRuns.Get(resourceGroup, factoryName, runId);
logging.LogInformation("Checking ADF pipeline status.");
//Create simple status for Data Factory Until comparison checks
string simpleStatus;
if (pipelineRun.Status == "InProgress")
{
simpleStatus = "Running";
}
else
{
simpleStatus = "Done";
}
logging.LogInformation("ADF pipeline status: " + pipelineRun.Status);
//Final return detail
outputString = "{ \"PipelineName\": \"" + pipelineName +
"\", \"RunId\": \"" + pipelineRun.RunId +
"\", \"SimpleStatus\": \"" + simpleStatus +
"\", \"Status\": \"" + pipelineRun.Status +
"\" }";
}
#endregion
JObject outputJson = JObject.Parse(outputString);
return(outputJson);
}
public static async Task <IActionResult> Run(
[HttpTrigger(AuthorizationLevel.Function, "get", "post", Route = null)] HttpRequest req,
ILogger log)
{
log.LogInformation("CheckPipelineStatus Function triggered by HTTP request.");
#region ParseRequestBody
log.LogInformation("Parsing body from request.");
string requestBody = await new StreamReader(req.Body).ReadToEndAsync();
dynamic data = JsonConvert.DeserializeObject(requestBody);
string outputString = string.Empty;
string tenantId = data?.tenantId;
string applicationId = data?.applicationId;
string authenticationKey = data?.authenticationKey;
string subscriptionId = data?.subscriptionId;
string resourceGroup = data?.resourceGroup;
string factoryName = data?.factoryName;
string pipelineName = data?.pipelineName;
string runId = data?.runId;
//Check body for values
if (
tenantId == null ||
applicationId == null ||
authenticationKey == null ||
subscriptionId == null ||
resourceGroup == null ||
factoryName == null ||
pipelineName == null ||
runId == null
)
{
log.LogInformation("Invalid body.");
return(new BadRequestObjectResult("Invalid request body, value(s) missing."));
}
#endregion
#region ResolveKeyVaultValues
log.LogInformation(RequestHelper.CheckGuid(applicationId).ToString());
if (!RequestHelper.CheckGuid(applicationId) && RequestHelper.CheckUri(applicationId))
{
log.LogInformation("Getting applicationId from Key Vault");
applicationId = KeyVaultClient.GetSecretFromUri(applicationId);
}
if (RequestHelper.CheckUri(authenticationKey))
{
log.LogInformation("Getting authenticationKey from Key Vault");
authenticationKey = KeyVaultClient.GetSecretFromUri(authenticationKey);
}
#endregion
#region GetPipelineStatus
//Create a data factory management client
log.LogInformation("Creating ADF connectivity client.");
using (var adfClient = DataFactoryClient.CreateDataFactoryClient(tenantId, applicationId, authenticationKey, subscriptionId))
{
//Get pipeline status with provided run id
PipelineRun pipelineRun;
pipelineRun = adfClient.PipelineRuns.Get(resourceGroup, factoryName, runId);
log.LogInformation("Checking ADF pipeline status.");
//Create simple status for Data Factory Until comparison checks
string simpleStatus;
if (pipelineRun.Status == "InProgress")
{
simpleStatus = "Running";
}
else
{
simpleStatus = "Done";
}
log.LogInformation("ADF pipeline status: " + pipelineRun.Status);
//Final return detail
outputString = "{ \"PipelineName\": \"" + pipelineName +
"\", \"RunId\": \"" + pipelineRun.RunId +
"\", \"SimpleStatus\": \"" + simpleStatus +
"\", \"Status\": \"" + pipelineRun.Status +
"\" }";
}
#endregion
JObject outputJson = JObject.Parse(outputString);
log.LogInformation("CheckPipelineStatus Function complete.");
return(new OkObjectResult(outputJson));
}
public static void QueryPipelineRuns(string subscriptionId, string resourceGroup, string factoryName, string _rungroupid, DateTime startDT, DateTime endDT, Logging logging)
{
#region QueryPipelineRuns
logging.LogInformation("Query ADF Pipeline Runs.");
string outputString = string.Empty;
DataTable dt = new DataTable();
dt.Columns.Add(new DataColumn("ExecutionUid", typeof(Guid)));
dt.Columns.Add(new DataColumn("TaskInstanceId", typeof(Int64)));
dt.Columns.Add(new DataColumn("TaskMasterId", typeof(Int64)));
//dt.Columns.Add(new DataColumn("AdditionalProperties", typeof(String)));
dt.Columns.Add(new DataColumn("DurationInMs", typeof(Int64)));
//dt.Columns.Add(new DataColumn("InvokedBy", typeof(String)));
dt.Columns.Add(new DataColumn("IsLastest", typeof(Boolean)));
dt.Columns.Add(new DataColumn("LastUpdated", typeof(DateTime)));
//dt.Columns.Add(new DataColumn("Message", typeof(String)));
//dt.Columns.Add(new DataColumn("Parameters", typeof(String)));
dt.Columns.Add(new DataColumn("RunId", typeof(Guid)));
//dt.Columns.Add(new DataColumn("RunGroupId", typeof(Guid)));
dt.Columns.Add(new DataColumn("PipelineName", typeof(String)));
dt.Columns.Add(new DataColumn("RunStart", typeof(DateTime)));
dt.Columns.Add(new DataColumn("RunEnd", typeof(DateTime)));
dt.Columns.Add(new DataColumn("RunDimensions", typeof(String)));
dt.Columns.Add(new DataColumn("Status", typeof(String)));
DataTable ActivityDt = new DataTable();
ActivityDt.Columns.Add(new DataColumn("ActivityName", typeof(String)));
ActivityDt.Columns.Add(new DataColumn("RunId", typeof(Guid)));
ActivityDt.Columns.Add(new DataColumn("ActivityRunStart", typeof(DateTime)));
ActivityDt.Columns.Add(new DataColumn("ActivityRunEnd", typeof(DateTime)));
ActivityDt.Columns.Add(new DataColumn("ActivityRunId", typeof(Guid)));
ActivityDt.Columns.Add(new DataColumn("ActivityType", typeof(String)));
//dt.Columns.Add(new DataColumn("AdditionalProperties", typeof(String)));
ActivityDt.Columns.Add(new DataColumn("DurationInMs", typeof(Int64)));
//dt.Columns.Add(new DataColumn("Error", typeof(String)));
//dt.Columns.Add(new DataColumn("Input", typeof(String)));
//dt.Columns.Add(new DataColumn("LinkedServiceName", typeof(String)));
ActivityDt.Columns.Add(new DataColumn("OutPut", typeof(String)));
ActivityDt.Columns.Add(new DataColumn("PipelineName", typeof(String)));
ActivityDt.Columns.Add(new DataColumn("PipelineRunId", typeof(String)));
ActivityDt.Columns.Add(new DataColumn("Status", typeof(String)));
using (var client = DataFactoryClient.CreateDataFactoryClient(subscriptionId))
{
//Get pipeline status with provided run id
PipelineRunsQueryResponse pipelineRunsQueryResponse;
RunFilterParameters filterParameterActivityRuns = new RunFilterParameters();
filterParameterActivityRuns.LastUpdatedAfter = startDT;
filterParameterActivityRuns.LastUpdatedBefore = endDT.AddHours(+2);
RunFilterParameters filterParameter = new RunFilterParameters();
filterParameter.LastUpdatedAfter = startDT;
filterParameter.LastUpdatedBefore = endDT;
IList <string> rungroupid = new List <string> {
_rungroupid
};
IList <RunQueryFilter> filter = new List <RunQueryFilter>();
filter.Add(new RunQueryFilter
{
Operand = RunQueryFilterOperand.RunGroupId,
OperatorProperty = RunQueryFilterOperator.Equals,
Values = rungroupid
});
filterParameter.Filters = filter;
logging.LogInformation(String.Format("API PipelineRuns.QueryByFactory Start"));
pipelineRunsQueryResponse = client.PipelineRuns.QueryByFactory(resourceGroup, factoryName, filterParameter);
logging.LogInformation(String.Format("API PipelineRuns.QueryByFactory End"));
var enumerator = pipelineRunsQueryResponse.Value.GetEnumerator();
PipelineRun pipelineRuns;
string runId = String.Empty;
int item = 0;
while (true)
{
for (bool hasMoreRuns = enumerator.MoveNext(); hasMoreRuns;)
{
pipelineRuns = enumerator.Current;
hasMoreRuns = enumerator.MoveNext();
runId = pipelineRuns.RunId;
item += 1;
logging.LogInformation(String.Format("PipelineRuns.QueryByFactory RunId {0} Current Item {1} of {2}", runId, item, pipelineRunsQueryResponse.Value.Count));
DataRow dr = dt.NewRow();
string _param = string.Empty;
foreach (var element in pipelineRuns.Parameters)
{
_param = element.Value;
break;
}
dr["ExecutionUid"] = Shared.JsonHelpers.GetStringValueFromJSON(logging, "ExecutionUid", JObject.Parse(_param), null, true);
dr["TaskInstanceId"] = Shared.JsonHelpers.GetStringValueFromJSON(logging, "TaskInstanceId", JObject.Parse(_param), null, true);
dr["TaskMasterId"] = Shared.JsonHelpers.GetStringValueFromJSON(logging, "TaskMasterId", JObject.Parse(_param), null, true);
//dr["AdditionalProperties"] = pipelineRuns.AdditionalProperties ?? (object)DBNull.Value;
dr["DurationInMs"] = pipelineRuns.DurationInMs ?? (object)DBNull.Value;
//dr["InvokedBy"] = pipelineRuns.InvokedBy ?? (object)DBNull.Value;
dr["IsLastest"] = pipelineRuns.IsLatest ?? (object)DBNull.Value;
dr["LastUpdated"] = pipelineRuns.LastUpdated ?? (object)DBNull.Value;
//dr["Message"] = pipelineRuns.Message ?? (object)DBNull.Value;
//dr["Parameters"] = _param;
dr["RunId"] = pipelineRuns.RunId;
//dr["RunGroupId"] = pipelineRuns.RunGroupId ?? (object)DBNull.Value;
dr["PipelineName"] = pipelineRuns.PipelineName ?? (object)DBNull.Value;
dr["RunStart"] = pipelineRuns.RunStart ?? (object)DBNull.Value;
dr["RunEnd"] = pipelineRuns.RunEnd ?? (object)DBNull.Value;
dr["RunDimensions"] = pipelineRuns.PipelineName ?? (object)DBNull.Value;
dr["Status"] = pipelineRuns.Status ?? (object)DBNull.Value;
dt.Rows.Add(dr);
QueryActivityRuns(subscriptionId, resourceGroup, factoryName, runId, runId, filterParameterActivityRuns, logging, ref ActivityDt);
}
if (pipelineRunsQueryResponse.ContinuationToken == null)
{
break;
}
filterParameter.ContinuationToken = pipelineRunsQueryResponse.ContinuationToken;
pipelineRunsQueryResponse = client.PipelineRuns.QueryByFactory(resourceGroup, factoryName, filterParameter);
enumerator = pipelineRunsQueryResponse.Value.GetEnumerator();
item = 0;
}
}
if (ActivityDt.Rows.Count > 0)
{
string TempTableName = "#Temp_ADFActivities_" + Guid.NewGuid().ToString();
TaskMetaDataDatabase TMD = new TaskMetaDataDatabase();
TMD.AutoBulkInsertAndMerge(ActivityDt, TempTableName, "ADFActivity");
}
if (dt.Rows.Count > 0)
{
string TempTableName = "#Temp_ADFPipelineRun_" + Guid.NewGuid().ToString();
TaskMetaDataDatabase TMD = new TaskMetaDataDatabase();
TMD.AutoBulkInsertAndMerge(dt, TempTableName, "ADFPipelineRun");
}
#endregion
}
public static JObject ExecutePipelineMethod(string subscriptionId, string resourceGroup, string factoryName, string pipelineName, Dictionary <string, object> parameters, Logging logging)
{
#region CreatePipelineRun
//Create a data factory management client
logging.LogInformation("Creating ADF connectivity client.");
string outputString = string.Empty;
using (var client = DataFactoryClient.CreateDataFactoryClient(subscriptionId))
{
//Run pipeline
CreateRunResponse runResponse;
PipelineRun pipelineRun;
if (parameters.Count == 0)
{
logging.LogInformation("Called pipeline without parameters.");
runResponse = client.Pipelines.CreateRunWithHttpMessagesAsync(
resourceGroup, factoryName, pipelineName).Result.Body;
}
else
{
logging.LogInformation("Called pipeline with parameters.");
logging.LogInformation("Number of parameters provided: " + parameters.Count);
System.Threading.Thread.Sleep(1000);
runResponse = client.Pipelines.CreateRunWithHttpMessagesAsync(
resourceGroup, factoryName, pipelineName, parameters: parameters).Result.Body;
runResponse = new CreateRunResponse();
runResponse.RunId = Guid.NewGuid().ToString();
}
logging.LogInformation("Pipeline run ID: " + runResponse.RunId);
//Wait and check for pipeline to start...
logging.LogInformation("Checking ADF pipeline status.");
//while (true)
//{
// pipelineRun = client.PipelineRuns.Get(
// resourceGroup, factoryName, runResponse.RunId);
// logging.LogInformation("ADF pipeline status: " + pipelineRun.Status);
//if (pipelineRun.Status == "Queued")
// System.Threading.Thread.Sleep(15000);
//else
// break;
//}
//Final return detail
outputString = "{ \"PipelineName\": \"" + pipelineName +
"\", \"RunId\": \"" + runResponse.RunId +
"\", \"Status\": \"" + "InProgress" + //pipelineRun.Status +
"\" }";
}
#endregion
JObject outputJson = JObject.Parse(outputString);
return(outputJson);
}
public static void QueryActivityRuns(string subscriptionId, string resourceGroup, string factoryName, string runId, string parentRunId, RunFilterParameters filterParameterActivityRuns, Logging logging, ref DataTable dt)
{
#region QueryActivityRuns
logging.LogInformation(String.Format("QueryActivityRuns - RunId {0}", runId));
string outputString = string.Empty;
using (var client = DataFactoryClient.CreateDataFactoryClient(subscriptionId))
{
//Get pipeline status with provided run id
logging.LogInformation(String.Format("QueryActivityRuns - RunId {0} - API QueryByPipelineRun Start", runId));
ActivityRunsQueryResponse activityRunsQueryResponse = client.ActivityRuns.QueryByPipelineRun(resourceGroup, factoryName, runId, filterParameterActivityRuns);
logging.LogInformation(String.Format("QueryActivityRuns - RunId {0} - API QueryByPipelineRun End", runId));
var enumeratorActivity = activityRunsQueryResponse.Value.GetEnumerator();
ActivityRun activityRun;
for (bool hasMoreActivityRuns = enumeratorActivity.MoveNext(); hasMoreActivityRuns;)
{
activityRun = enumeratorActivity.Current;
hasMoreActivityRuns = enumeratorActivity.MoveNext();
DataRow dr = dt.NewRow();
dr["ActivityName"] = activityRun.ActivityName ?? (object)DBNull.Value;
dr["RunId"] = parentRunId;
dr["ActivityRunStart"] = activityRun.ActivityRunStart ?? (object)DBNull.Value;
dr["ActivityRunEnd"] = activityRun.ActivityRunEnd ?? (object)DBNull.Value;
dr["ActivityRunId"] = activityRun.ActivityRunId ?? (object)DBNull.Value;
dr["ActivityType"] = activityRun.ActivityType ?? (object)DBNull.Value;
//dr["AdditionalProperties"] = activityRun.AdditionalProperties ?? (object)DBNull.Value;
dr["DurationInMs"] = activityRun.DurationInMs ?? (object)DBNull.Value;
//dr["Error"] = activityRun.Error ?? (object)DBNull.Value;
//dr["Input"] = activityRun.Input ?? (object)DBNull.Value;
//dr["LinkedServiceName"] = activityRun.LinkedServiceName ?? (object)DBNull.Value;
dr["OutPut"] = activityRun.Output ?? (object)DBNull.Value;
dr["PipelineName"] = activityRun.PipelineName ?? (object)DBNull.Value;
dr["PipelineRunId"] = activityRun.PipelineRunId ?? (object)DBNull.Value;
dr["Status"] = activityRun.Status ?? (object)DBNull.Value;
dt.Rows.Add(dr);
if (activityRun.ActivityType == "ExecutePipeline")
{
string _runId = Shared.JsonHelpers.GetStringValueFromJSON(logging, "pipelineRunId", (JObject)activityRun.Output, null, true);
if (!String.IsNullOrEmpty(_runId))
{
QueryActivityRuns(subscriptionId, resourceGroup, factoryName, _runId, parentRunId, filterParameterActivityRuns, logging, ref dt);
}
else
{
logging.LogInformation(String.Format("QueryActivityRuns - RunId Is null for {0} ", (JObject)activityRun.Output));
}
}
}
}
#endregion
}
public static async Task <IActionResult> Run(
[HttpTrigger(AuthorizationLevel.Function, "get", "post", Route = null)] HttpRequest req,
ILogger log)
{
log.LogInformation("ValidatePipeline Function triggered by HTTP request.");
#region ParseRequestBody
log.LogInformation("Parsing body from request.");
string requestBody = await new StreamReader(req.Body).ReadToEndAsync();
dynamic data = JsonConvert.DeserializeObject(requestBody);
string outputString = string.Empty;
string tenantId = data?.tenantId;
string applicationId = data?.applicationId;
string authenticationKey = data?.authenticationKey;
string subscriptionId = data?.subscriptionId;
string resourceGroup = data?.resourceGroup;
string factoryName = data?.factoryName;
string pipelineName = data?.pipelineName;
//Check body for values
if (
tenantId == null ||
applicationId == null ||
authenticationKey == null ||
subscriptionId == null ||
resourceGroup == null ||
factoryName == null ||
pipelineName == null
)
{
log.LogInformation("Invalid body.");
return(new BadRequestObjectResult("Invalid request body, value(s) missing."));
}
#endregion
#region ResolveKeyVaultValues
log.LogInformation(RequestHelper.CheckGuid(applicationId).ToString());
if (!RequestHelper.CheckGuid(applicationId) && RequestHelper.CheckUri(applicationId))
{
log.LogInformation("Getting applicationId from Key Vault");
applicationId = KeyVaultClient.GetSecretFromUri(applicationId);
}
if (RequestHelper.CheckUri(authenticationKey))
{
log.LogInformation("Getting authenticationKey from Key Vault");
authenticationKey = KeyVaultClient.GetSecretFromUri(authenticationKey);
}
#endregion
#region ValidatePipeline
//Create a data factory management client
log.LogInformation("Creating ADF connectivity client.");
using (var adfClient = DataFactoryClient.CreateDataFactoryClient(tenantId, applicationId, authenticationKey, subscriptionId))
{
PipelineResource pipelineResource;
try
{
pipelineResource = adfClient.Pipelines.Get(resourceGroup, factoryName, pipelineName);
log.LogInformation(pipelineResource.Id.ToString());
log.LogInformation(pipelineResource.Name.ToString());
outputString = "{ \"PipelineExists\": \"" + true.ToString() +
"\", \"PipelineName\": \"" + pipelineResource.Name.ToString() +
"\", \"PipelineId\": \"" + pipelineResource.Id.ToString() +
"\", \"PipelineType\": \"" + pipelineResource.Type.ToString() +
"\", \"ActivityCount\": \"" + pipelineResource.Activities.Count.ToString() +
"\" }";
}
catch (Exception ex)
{
outputString = "{ \"PipelineExists\": \"" + false.ToString() +
"\", \"ProvidedPipelineName\": \"" + pipelineName +
"\" }";
log.LogInformation(ex.Message);
}
}
#endregion
JObject outputJson = JObject.Parse(outputString);
log.LogInformation("CheckPipelineStatus Function complete.");
//return false positive so PipelineExists result can be evaluated by Data Factory
return(new OkObjectResult(outputJson));
}
public static async Task <IActionResult> Run(
[HttpTrigger(AuthorizationLevel.Function, "post", Route = null)] HttpRequest req,
ILogger log)
{
log.LogInformation("ExecutePipeline Function triggered by HTTP request.");
#region ParseRequestBody
log.LogInformation("Parsing body from request.");
string requestBody = await new StreamReader(req.Body).ReadToEndAsync();
dynamic data = JsonConvert.DeserializeObject(requestBody);
string tenantId = data?.tenantId;
string applicationId = data?.applicationId;
string authenticationKey = data?.authenticationKey;
string subscriptionId = data?.subscriptionId;
string resourceGroup = data?.resourceGroup;
string factoryName = data?.factoryName;
string pipelineName = data?.pipelineName;
//Check body for values
if (
tenantId == null ||
applicationId == null ||
authenticationKey == null ||
subscriptionId == null ||
resourceGroup == null ||
factoryName == null ||
pipelineName == null
)
{
log.LogInformation("Invalid body.");
return(new BadRequestObjectResult("Invalid request body, value missing."));
}
#endregion
#region ResolveKeyVaultValues
if (!RequestHelper.CheckGuid(applicationId) && RequestHelper.CheckUri(applicationId))
{
log.LogInformation("Getting applicationId from Key Vault");
applicationId = KeyVaultClient.GetSecretFromUri(applicationId);
}
if (RequestHelper.CheckUri(authenticationKey))
{
log.LogInformation("Getting authenticationKey from Key Vault");
authenticationKey = KeyVaultClient.GetSecretFromUri(authenticationKey);
}
#endregion
#region CreatePipelineRun
//Create a data factory management client
log.LogInformation("Creating ADF connectivity client.");
string outputString = string.Empty;
using (var client = DataFactoryClient.CreateDataFactoryClient(tenantId, applicationId, authenticationKey, subscriptionId))
{
//Run pipeline
CreateRunResponse runResponse;
PipelineRun pipelineRun;
if (data?.pipelineParameters == null)
{
log.LogInformation("Called pipeline without parameters.");
runResponse = client.Pipelines.CreateRunWithHttpMessagesAsync(
resourceGroup, factoryName, pipelineName).Result.Body;
}
else
{
log.LogInformation("Called pipeline with parameters.");
JObject jObj = JObject.Parse(requestBody);
Dictionary <string, object> parameters = jObj["pipelineParameters"].ToObject <Dictionary <string, object> >();
log.LogInformation("Number of parameters provided: " + jObj.Count.ToString());
runResponse = client.Pipelines.CreateRunWithHttpMessagesAsync(
resourceGroup, factoryName, pipelineName, parameters: parameters).Result.Body;
}
log.LogInformation("Pipeline run ID: " + runResponse.RunId);
//Wait and check for pipeline to start...
log.LogInformation("Checking ADF pipeline status.");
while (true)
{
pipelineRun = client.PipelineRuns.Get(
resourceGroup, factoryName, runResponse.RunId);
log.LogInformation("ADF pipeline status: " + pipelineRun.Status);
if (pipelineRun.Status == "Queued")
{
System.Threading.Thread.Sleep(15000);
}
else
{
break;
}
}
//Final return detail
outputString = "{ \"PipelineName\": \"" + pipelineName +
"\", \"RunId\": \"" + pipelineRun.RunId +
"\", \"Status\": \"" + pipelineRun.Status +
"\" }";
}
#endregion
JObject outputJson = JObject.Parse(outputString);
log.LogInformation("ExecutePipeline Function complete.");
return(new OkObjectResult(outputJson));
}
public static async Task <IActionResult> Run(
[HttpTrigger(AuthorizationLevel.Function, "get", "post", Route = null)] HttpRequest req,
ILogger log)
{
log.LogInformation("GetActivityErrors Function triggered by HTTP request.");
#region ParseRequestBody
log.LogInformation("Parsing body from request.");
string requestBody = await new StreamReader(req.Body).ReadToEndAsync();
dynamic inputData = JsonConvert.DeserializeObject(requestBody);
string tenantId = inputData?.tenantId;
string applicationId = inputData?.applicationId;
string authenticationKey = inputData?.authenticationKey;
string subscriptionId = inputData?.subscriptionId;
string resourceGroup = inputData?.resourceGroup;
string factoryName = inputData?.factoryName;
string pipelineName = inputData?.pipelineName;
string runId = inputData?.runId;
//Check body for values
if (
tenantId == null ||
applicationId == null ||
authenticationKey == null ||
subscriptionId == null ||
resourceGroup == null ||
factoryName == null ||
pipelineName == null ||
runId == null
)
{
log.LogInformation("Invalid body.");
return(new BadRequestObjectResult("Invalid request body, value(s) missing."));
}
#endregion
//Query and output support variables
int daysOfRuns = 7; //max duration for mandatory RunFilterParameters
DateTime today = DateTime.Now;
DateTime lastWeek = DateTime.Now.AddDays(-daysOfRuns);
dynamic outputBlock = new JObject();
dynamic outputBlockInner;
//Create a data factory management client
log.LogInformation("Creating ADF connectivity client.");
using (var client = DataFactoryClient.CreateDataFactoryClient(tenantId, applicationId, authenticationKey, subscriptionId))
{
#region SetPipelineRunDetails
//Get pipeline details
PipelineRun pipelineRun;
pipelineRun = client.PipelineRuns.Get(resourceGroup, factoryName, runId);
log.LogInformation("Querying ADF pipeline for Activity Runs.");
RunFilterParameters filterParams = new RunFilterParameters(lastWeek, today);
ActivityRunsQueryResponse queryResponse = client.ActivityRuns.QueryByPipelineRun(resourceGroup, factoryName, runId, filterParams);
//Create initial output content
outputBlock.PipelineName = pipelineName;
outputBlock.PipelineStatus = pipelineRun.Status;
outputBlock.RunId = runId;
outputBlock.ResponseCount = queryResponse.Value.Count;
outputBlock.ResponseErrorCount = 0;
outputBlock.Errors = new JArray();
JObject errorDetails;
#endregion
log.LogInformation("Pipeline status: " + pipelineRun.Status);
log.LogInformation("Activities found in pipeline response: " + queryResponse.Value.Count.ToString());
#region GetActivityDetails
//Loop over activities in pipeline run
foreach (var activity in queryResponse.Value)
{
if (string.IsNullOrEmpty(activity.Error.ToString()))
{
continue; //just incase
}
//Parse error output to customise output
outputBlockInner = JsonConvert.DeserializeObject(activity.Error.ToString());
string errorCode = outputBlockInner?.errorCode;
string errorType = outputBlockInner?.failureType;
string errorMessage = outputBlockInner?.message;
//Get output details
if (!string.IsNullOrEmpty(errorCode))
{
log.LogInformation("Activity run id: " + activity.ActivityRunId);
log.LogInformation("Activity name: " + activity.ActivityName);
log.LogInformation("Activity type: " + activity.ActivityType);
log.LogInformation("Error message: " + errorMessage);
outputBlock.ResponseErrorCount += 1;
//Construct custom error information block
errorDetails = JObject.Parse("{ \"ActivityRunId\": \"" + activity.ActivityRunId +
"\", \"ActivityName\": \"" + activity.ActivityName +
"\", \"ActivityType\": \"" + activity.ActivityType +
"\", \"ErrorCode\": \"" + errorCode +
"\", \"ErrorType\": \"" + errorType +
"\", \"ErrorMessage\": \"" + errorMessage +
"\" }");
outputBlock.Errors.Add(errorDetails);
}
}
#endregion
}
log.LogInformation("GetActivityErrors Function complete.");
return(new OkObjectResult(outputBlock));
}
