public static async Task <IActionResult> Run(
[HttpTrigger(AuthorizationLevel.Function, "get", "post", Route = null)] HttpRequest req,
ILogger log)
{
log.LogInformation("CheckPipelineStatus Function triggered by HTTP request.");
#region ParseRequestBody
log.LogInformation("Parsing body from request.");
string requestBody = await new StreamReader(req.Body).ReadToEndAsync();
dynamic data = JsonConvert.DeserializeObject(requestBody);
string outputString = string.Empty;
string tenantId = data?.tenantId;
string applicationId = data?.applicationId;
string authenticationKey = data?.authenticationKey;
string subscriptionId = data?.subscriptionId;
string resourceGroup = data?.resourceGroup;
string factoryName = data?.factoryName;
string pipelineName = data?.pipelineName;
string runId = data?.runId;
//Check body for values
if (
tenantId == null ||
applicationId == null ||
authenticationKey == null ||
subscriptionId == null ||
resourceGroup == null ||
factoryName == null ||
pipelineName == null ||
runId == null
)
{
log.LogInformation("Invalid body.");
return(new BadRequestObjectResult("Invalid request body, value(s) missing."));
}
#endregion
#region ResolveKeyVaultValues
log.LogInformation(RequestHelper.CheckGuid(applicationId).ToString());
if (!RequestHelper.CheckGuid(applicationId) && RequestHelper.CheckUri(applicationId))
{
log.LogInformation("Getting applicationId from Key Vault");
applicationId = KeyVaultClient.GetSecretFromUri(applicationId);
}
if (RequestHelper.CheckUri(authenticationKey))
{
log.LogInformation("Getting authenticationKey from Key Vault");
authenticationKey = KeyVaultClient.GetSecretFromUri(authenticationKey);
}
#endregion
#region GetPipelineStatus
//Create a data factory management client
log.LogInformation("Creating ADF connectivity client.");
using (var adfClient = DataFactoryClient.CreateDataFactoryClient(tenantId, applicationId, authenticationKey, subscriptionId))
{
//Get pipeline status with provided run id
PipelineRun pipelineRun;
pipelineRun = adfClient.PipelineRuns.Get(resourceGroup, factoryName, runId);
log.LogInformation("Checking ADF pipeline status.");
//Create simple status for Data Factory Until comparison checks
string simpleStatus;
if (pipelineRun.Status == "InProgress")
{
simpleStatus = "Running";
}
else
{
simpleStatus = "Done";
}
log.LogInformation("ADF pipeline status: " + pipelineRun.Status);
//Final return detail
outputString = "{ \"PipelineName\": \"" + pipelineName +
"\", \"RunId\": \"" + pipelineRun.RunId +
"\", \"SimpleStatus\": \"" + simpleStatus +
"\", \"Status\": \"" + pipelineRun.Status +
"\" }";
}
#endregion
JObject outputJson = JObject.Parse(outputString);
log.LogInformation("CheckPipelineStatus Function complete.");
return(new OkObjectResult(outputJson));
}
public static async Task <IActionResult> Run(
[HttpTrigger(AuthorizationLevel.Function, "post", Route = null)] HttpRequest req,
ILogger log)
{
log.LogInformation("ExecutePipeline Function triggered by HTTP request.");
#region ParseRequestBody
log.LogInformation("Parsing body from request.");
string requestBody = await new StreamReader(req.Body).ReadToEndAsync();
dynamic data = JsonConvert.DeserializeObject(requestBody);
string tenantId = data?.tenantId;
string applicationId = data?.applicationId;
string authenticationKey = data?.authenticationKey;
string subscriptionId = data?.subscriptionId;
string resourceGroup = data?.resourceGroup;
string factoryName = data?.factoryName;
string pipelineName = data?.pipelineName;
//Check body for values
if (
tenantId == null ||
applicationId == null ||
authenticationKey == null ||
subscriptionId == null ||
resourceGroup == null ||
factoryName == null ||
pipelineName == null
)
{
log.LogInformation("Invalid body.");
return(new BadRequestObjectResult("Invalid request body, value missing."));
}
#endregion
#region ResolveKeyVaultValues
if (!RequestHelper.CheckGuid(applicationId) && RequestHelper.CheckUri(applicationId))
{
log.LogInformation("Getting applicationId from Key Vault");
applicationId = KeyVaultClient.GetSecretFromUri(applicationId);
}
if (RequestHelper.CheckUri(authenticationKey))
{
log.LogInformation("Getting authenticationKey from Key Vault");
authenticationKey = KeyVaultClient.GetSecretFromUri(authenticationKey);
}
#endregion
#region CreatePipelineRun
//Create a data factory management client
log.LogInformation("Creating ADF connectivity client.");
string outputString = string.Empty;
using (var client = DataFactoryClient.CreateDataFactoryClient(tenantId, applicationId, authenticationKey, subscriptionId))
{
//Run pipeline
CreateRunResponse runResponse;
PipelineRun pipelineRun;
if (data?.pipelineParameters == null)
{
log.LogInformation("Called pipeline without parameters.");
runResponse = client.Pipelines.CreateRunWithHttpMessagesAsync(
resourceGroup, factoryName, pipelineName).Result.Body;
}
else
{
log.LogInformation("Called pipeline with parameters.");
JObject jObj = JObject.Parse(requestBody);
Dictionary <string, object> parameters = jObj["pipelineParameters"].ToObject <Dictionary <string, object> >();
log.LogInformation("Number of parameters provided: " + jObj.Count.ToString());
runResponse = client.Pipelines.CreateRunWithHttpMessagesAsync(
resourceGroup, factoryName, pipelineName, parameters: parameters).Result.Body;
}
log.LogInformation("Pipeline run ID: " + runResponse.RunId);
//Wait and check for pipeline to start...
log.LogInformation("Checking ADF pipeline status.");
while (true)
{
pipelineRun = client.PipelineRuns.Get(
resourceGroup, factoryName, runResponse.RunId);
log.LogInformation("ADF pipeline status: " + pipelineRun.Status);
if (pipelineRun.Status == "Queued")
{
System.Threading.Thread.Sleep(15000);
}
else
{
break;
}
}
//Final return detail
outputString = "{ \"PipelineName\": \"" + pipelineName +
"\", \"RunId\": \"" + pipelineRun.RunId +
"\", \"Status\": \"" + pipelineRun.Status +
"\" }";
}
#endregion
JObject outputJson = JObject.Parse(outputString);
log.LogInformation("ExecutePipeline Function complete.");
return(new OkObjectResult(outputJson));
}
public static async Task <IActionResult> Run(
[HttpTrigger(AuthorizationLevel.Function, "get", "post", Route = null)] HttpRequest req,
ILogger log)
{
log.LogInformation("ValidatePipeline Function triggered by HTTP request.");
#region ParseRequestBody
log.LogInformation("Parsing body from request.");
string requestBody = await new StreamReader(req.Body).ReadToEndAsync();
dynamic data = JsonConvert.DeserializeObject(requestBody);
string outputString = string.Empty;
string tenantId = data?.tenantId;
string applicationId = data?.applicationId;
string authenticationKey = data?.authenticationKey;
string subscriptionId = data?.subscriptionId;
string resourceGroup = data?.resourceGroup;
string factoryName = data?.factoryName;
string pipelineName = data?.pipelineName;
//Check body for values
if (
tenantId == null ||
applicationId == null ||
authenticationKey == null ||
subscriptionId == null ||
resourceGroup == null ||
factoryName == null ||
pipelineName == null
)
{
log.LogInformation("Invalid body.");
return(new BadRequestObjectResult("Invalid request body, value(s) missing."));
}
#endregion
#region ResolveKeyVaultValues
log.LogInformation(RequestHelper.CheckGuid(applicationId).ToString());
if (!RequestHelper.CheckGuid(applicationId) && RequestHelper.CheckUri(applicationId))
{
log.LogInformation("Getting applicationId from Key Vault");
applicationId = KeyVaultClient.GetSecretFromUri(applicationId);
}
if (RequestHelper.CheckUri(authenticationKey))
{
log.LogInformation("Getting authenticationKey from Key Vault");
authenticationKey = KeyVaultClient.GetSecretFromUri(authenticationKey);
}
#endregion
#region ValidatePipeline
//Create a data factory management client
log.LogInformation("Creating ADF connectivity client.");
using (var adfClient = DataFactoryClient.CreateDataFactoryClient(tenantId, applicationId, authenticationKey, subscriptionId))
{
PipelineResource pipelineResource;
try
{
pipelineResource = adfClient.Pipelines.Get(resourceGroup, factoryName, pipelineName);
log.LogInformation(pipelineResource.Id.ToString());
log.LogInformation(pipelineResource.Name.ToString());
outputString = "{ \"PipelineExists\": \"" + true.ToString() +
"\", \"PipelineName\": \"" + pipelineResource.Name.ToString() +
"\", \"PipelineId\": \"" + pipelineResource.Id.ToString() +
"\", \"PipelineType\": \"" + pipelineResource.Type.ToString() +
"\", \"ActivityCount\": \"" + pipelineResource.Activities.Count.ToString() +
"\" }";
}
catch (Exception ex)
{
outputString = "{ \"PipelineExists\": \"" + false.ToString() +
"\", \"ProvidedPipelineName\": \"" + pipelineName +
"\" }";
log.LogInformation(ex.Message);
}
}
#endregion
JObject outputJson = JObject.Parse(outputString);
log.LogInformation("CheckPipelineStatus Function complete.");
//return false positive so PipelineExists result can be evaluated by Data Factory
return(new OkObjectResult(outputJson));
}
public static async Task <IActionResult> Run(
[HttpTrigger(AuthorizationLevel.Function, "get", "post", Route = null)] HttpRequest req,
ILogger log)
{
log.LogInformation("GetActivityErrors Function triggered by HTTP request.");
#region ParseRequestBody
log.LogInformation("Parsing body from request.");
string requestBody = await new StreamReader(req.Body).ReadToEndAsync();
dynamic inputData = JsonConvert.DeserializeObject(requestBody);
string tenantId = inputData?.tenantId;
string applicationId = inputData?.applicationId;
string authenticationKey = inputData?.authenticationKey;
string subscriptionId = inputData?.subscriptionId;
string resourceGroup = inputData?.resourceGroup;
string factoryName = inputData?.factoryName;
string pipelineName = inputData?.pipelineName;
string runId = inputData?.runId;
//Check body for values
if (
tenantId == null ||
applicationId == null ||
authenticationKey == null ||
subscriptionId == null ||
resourceGroup == null ||
factoryName == null ||
pipelineName == null ||
runId == null
)
{
log.LogInformation("Invalid body.");
return(new BadRequestObjectResult("Invalid request body, value(s) missing."));
}
#endregion
#region ResolveKeyVaultValues
if (!RequestHelper.CheckGuid(applicationId) && RequestHelper.CheckUri(applicationId))
{
log.LogInformation("Getting applicationId from Key Vault");
applicationId = KeyVaultClient.GetSecretFromUri(applicationId);
}
if (RequestHelper.CheckUri(authenticationKey))
{
log.LogInformation("Getting authenticationKey from Key Vault");
authenticationKey = KeyVaultClient.GetSecretFromUri(authenticationKey);
}
#endregion
//Query and output support variables
int daysOfRuns = 7; //max duration for mandatory RunFilterParameters
DateTime today = DateTime.Now;
DateTime lastWeek = DateTime.Now.AddDays(-daysOfRuns);
dynamic outputBlock = new JObject();
dynamic outputBlockInner;
//Create a data factory management client
log.LogInformation("Creating ADF connectivity client.");
using (var client = DataFactoryClient.CreateDataFactoryClient(tenantId, applicationId, authenticationKey, subscriptionId))
{
#region SetPipelineRunDetails
//Get pipeline details
PipelineRun pipelineRun;
pipelineRun = client.PipelineRuns.Get(resourceGroup, factoryName, runId);
log.LogInformation("Querying ADF pipeline for Activity Runs.");
RunFilterParameters filterParams = new RunFilterParameters(lastWeek, today);
ActivityRunsQueryResponse queryResponse = client.ActivityRuns.QueryByPipelineRun(resourceGroup, factoryName, runId, filterParams);
//Create initial output content
outputBlock.PipelineName = pipelineName;
outputBlock.PipelineStatus = pipelineRun.Status;
outputBlock.RunId = runId;
outputBlock.ResponseCount = queryResponse.Value.Count;
outputBlock.ResponseErrorCount = 0;
outputBlock.Errors = new JArray();
JObject errorDetails;
#endregion
log.LogInformation("Pipeline status: " + pipelineRun.Status);
log.LogInformation("Activities found in pipeline response: " + queryResponse.Value.Count.ToString());
#region GetActivityDetails
//Loop over activities in pipeline run
foreach (var activity in queryResponse.Value)
{
if (string.IsNullOrEmpty(activity.Error.ToString()))
{
continue; //just incase
}
//Parse error output to customise output
outputBlockInner = JsonConvert.DeserializeObject(activity.Error.ToString());
string errorCode = outputBlockInner?.errorCode;
string errorType = outputBlockInner?.failureType;
string errorMessage = outputBlockInner?.message;
//Get output details
if (!string.IsNullOrEmpty(errorCode))
{
log.LogInformation("Activity run id: " + activity.ActivityRunId);
log.LogInformation("Activity name: " + activity.ActivityName);
log.LogInformation("Activity type: " + activity.ActivityType);
log.LogInformation("Error message: " + errorMessage);
outputBlock.ResponseErrorCount += 1;
//Construct custom error information block
errorDetails = JObject.Parse("{ \"ActivityRunId\": \"" + activity.ActivityRunId +
"\", \"ActivityName\": \"" + activity.ActivityName +
"\", \"ActivityType\": \"" + activity.ActivityType +
"\", \"ErrorCode\": \"" + errorCode +
"\", \"ErrorType\": \"" + errorType +
"\", \"ErrorMessage\": \"" + errorMessage +
"\" }");
outputBlock.Errors.Add(errorDetails);
}
}
#endregion
}
log.LogInformation("GetActivityErrors Function complete.");
return(new OkObjectResult(outputBlock));
}
