public static void QueryPipelineRuns(string subscriptionId, string resourceGroup, string factoryName, string _rungroupid, DateTime startDT, DateTime endDT, Logging logging)
{
#region QueryPipelineRuns
logging.LogInformation("Query ADF Pipeline Runs.");
string outputString = string.Empty;
DataTable dt = new DataTable();
dt.Columns.Add(new DataColumn("ExecutionUid", typeof(Guid)));
dt.Columns.Add(new DataColumn("TaskInstanceId", typeof(Int64)));
dt.Columns.Add(new DataColumn("TaskMasterId", typeof(Int64)));
//dt.Columns.Add(new DataColumn("AdditionalProperties", typeof(String)));
dt.Columns.Add(new DataColumn("DurationInMs", typeof(Int64)));
//dt.Columns.Add(new DataColumn("InvokedBy", typeof(String)));
dt.Columns.Add(new DataColumn("IsLastest", typeof(Boolean)));
dt.Columns.Add(new DataColumn("LastUpdated", typeof(DateTime)));
//dt.Columns.Add(new DataColumn("Message", typeof(String)));
//dt.Columns.Add(new DataColumn("Parameters", typeof(String)));
dt.Columns.Add(new DataColumn("RunId", typeof(Guid)));
//dt.Columns.Add(new DataColumn("RunGroupId", typeof(Guid)));
dt.Columns.Add(new DataColumn("PipelineName", typeof(String)));
dt.Columns.Add(new DataColumn("RunStart", typeof(DateTime)));
dt.Columns.Add(new DataColumn("RunEnd", typeof(DateTime)));
dt.Columns.Add(new DataColumn("RunDimensions", typeof(String)));
dt.Columns.Add(new DataColumn("Status", typeof(String)));
DataTable ActivityDt = new DataTable();
ActivityDt.Columns.Add(new DataColumn("ActivityName", typeof(String)));
ActivityDt.Columns.Add(new DataColumn("RunId", typeof(Guid)));
ActivityDt.Columns.Add(new DataColumn("ActivityRunStart", typeof(DateTime)));
ActivityDt.Columns.Add(new DataColumn("ActivityRunEnd", typeof(DateTime)));
ActivityDt.Columns.Add(new DataColumn("ActivityRunId", typeof(Guid)));
ActivityDt.Columns.Add(new DataColumn("ActivityType", typeof(String)));
//dt.Columns.Add(new DataColumn("AdditionalProperties", typeof(String)));
ActivityDt.Columns.Add(new DataColumn("DurationInMs", typeof(Int64)));
//dt.Columns.Add(new DataColumn("Error", typeof(String)));
//dt.Columns.Add(new DataColumn("Input", typeof(String)));
//dt.Columns.Add(new DataColumn("LinkedServiceName", typeof(String)));
ActivityDt.Columns.Add(new DataColumn("OutPut", typeof(String)));
ActivityDt.Columns.Add(new DataColumn("PipelineName", typeof(String)));
ActivityDt.Columns.Add(new DataColumn("PipelineRunId", typeof(String)));
ActivityDt.Columns.Add(new DataColumn("Status", typeof(String)));
using (var client = DataFactoryClient.CreateDataFactoryClient(subscriptionId))
{
//Get pipeline status with provided run id
PipelineRunsQueryResponse pipelineRunsQueryResponse;
RunFilterParameters filterParameterActivityRuns = new RunFilterParameters();
filterParameterActivityRuns.LastUpdatedAfter = startDT;
filterParameterActivityRuns.LastUpdatedBefore = endDT.AddHours(+2);
RunFilterParameters filterParameter = new RunFilterParameters();
filterParameter.LastUpdatedAfter = startDT;
filterParameter.LastUpdatedBefore = endDT;
IList <string> rungroupid = new List <string> {
_rungroupid
};
IList <RunQueryFilter> filter = new List <RunQueryFilter>();
filter.Add(new RunQueryFilter
{
Operand = RunQueryFilterOperand.RunGroupId,
OperatorProperty = RunQueryFilterOperator.Equals,
Values = rungroupid
});
filterParameter.Filters = filter;
logging.LogInformation(String.Format("API PipelineRuns.QueryByFactory Start"));
pipelineRunsQueryResponse = client.PipelineRuns.QueryByFactory(resourceGroup, factoryName, filterParameter);
logging.LogInformation(String.Format("API PipelineRuns.QueryByFactory End"));
var enumerator = pipelineRunsQueryResponse.Value.GetEnumerator();
PipelineRun pipelineRuns;
string runId = String.Empty;
int item = 0;
while (true)
{
for (bool hasMoreRuns = enumerator.MoveNext(); hasMoreRuns;)
{
pipelineRuns = enumerator.Current;
hasMoreRuns = enumerator.MoveNext();
runId = pipelineRuns.RunId;
item += 1;
logging.LogInformation(String.Format("PipelineRuns.QueryByFactory RunId {0} Current Item {1} of {2}", runId, item, pipelineRunsQueryResponse.Value.Count));
DataRow dr = dt.NewRow();
string _param = string.Empty;
foreach (var element in pipelineRuns.Parameters)
{
_param = element.Value;
break;
}
dr["ExecutionUid"] = Shared.JsonHelpers.GetStringValueFromJSON(logging, "ExecutionUid", JObject.Parse(_param), null, true);
dr["TaskInstanceId"] = Shared.JsonHelpers.GetStringValueFromJSON(logging, "TaskInstanceId", JObject.Parse(_param), null, true);
dr["TaskMasterId"] = Shared.JsonHelpers.GetStringValueFromJSON(logging, "TaskMasterId", JObject.Parse(_param), null, true);
//dr["AdditionalProperties"] = pipelineRuns.AdditionalProperties ?? (object)DBNull.Value;
dr["DurationInMs"] = pipelineRuns.DurationInMs ?? (object)DBNull.Value;
//dr["InvokedBy"] = pipelineRuns.InvokedBy ?? (object)DBNull.Value;
dr["IsLastest"] = pipelineRuns.IsLatest ?? (object)DBNull.Value;
dr["LastUpdated"] = pipelineRuns.LastUpdated ?? (object)DBNull.Value;
//dr["Message"] = pipelineRuns.Message ?? (object)DBNull.Value;
//dr["Parameters"] = _param;
dr["RunId"] = pipelineRuns.RunId;
//dr["RunGroupId"] = pipelineRuns.RunGroupId ?? (object)DBNull.Value;
dr["PipelineName"] = pipelineRuns.PipelineName ?? (object)DBNull.Value;
dr["RunStart"] = pipelineRuns.RunStart ?? (object)DBNull.Value;
dr["RunEnd"] = pipelineRuns.RunEnd ?? (object)DBNull.Value;
dr["RunDimensions"] = pipelineRuns.PipelineName ?? (object)DBNull.Value;
dr["Status"] = pipelineRuns.Status ?? (object)DBNull.Value;
dt.Rows.Add(dr);
QueryActivityRuns(subscriptionId, resourceGroup, factoryName, runId, runId, filterParameterActivityRuns, logging, ref ActivityDt);
}
if (pipelineRunsQueryResponse.ContinuationToken == null)
{
break;
}
filterParameter.ContinuationToken = pipelineRunsQueryResponse.ContinuationToken;
pipelineRunsQueryResponse = client.PipelineRuns.QueryByFactory(resourceGroup, factoryName, filterParameter);
enumerator = pipelineRunsQueryResponse.Value.GetEnumerator();
item = 0;
}
}
if (ActivityDt.Rows.Count > 0)
{
string TempTableName = "#Temp_ADFActivities_" + Guid.NewGuid().ToString();
TaskMetaDataDatabase TMD = new TaskMetaDataDatabase();
TMD.AutoBulkInsertAndMerge(ActivityDt, TempTableName, "ADFActivity");
}
if (dt.Rows.Count > 0)
{
string TempTableName = "#Temp_ADFPipelineRun_" + Guid.NewGuid().ToString();
TaskMetaDataDatabase TMD = new TaskMetaDataDatabase();
TMD.AutoBulkInsertAndMerge(dt, TempTableName, "ADFPipelineRun");
}
#endregion
}
public virtual Response <TriggerRunsQueryResponse> QueryTriggerRunsByWorkspace(RunFilterParameters filterParameters, CancellationToken cancellationToken = default)
{
using var scope = _clientDiagnostics.CreateScope("TriggerRunClient.QueryTriggerRunsByWorkspace");
scope.Start();
try
{
return(RestClient.QueryTriggerRunsByWorkspace(filterParameters, cancellationToken));
}
catch (Exception e)
{
scope.Failed(e);
throw;
}
}
public static void QueryActivityRuns(string subscriptionId, string resourceGroup, string factoryName, string runId, string parentRunId, RunFilterParameters filterParameterActivityRuns, Logging logging, ref DataTable dt)
{
#region QueryActivityRuns
logging.LogInformation(String.Format("QueryActivityRuns - RunId {0}", runId));
string outputString = string.Empty;
using (var client = DataFactoryClient.CreateDataFactoryClient(subscriptionId))
{
//Get pipeline status with provided run id
logging.LogInformation(String.Format("QueryActivityRuns - RunId {0} - API QueryByPipelineRun Start", runId));
ActivityRunsQueryResponse activityRunsQueryResponse = client.ActivityRuns.QueryByPipelineRun(resourceGroup, factoryName, runId, filterParameterActivityRuns);
logging.LogInformation(String.Format("QueryActivityRuns - RunId {0} - API QueryByPipelineRun End", runId));
var enumeratorActivity = activityRunsQueryResponse.Value.GetEnumerator();
ActivityRun activityRun;
for (bool hasMoreActivityRuns = enumeratorActivity.MoveNext(); hasMoreActivityRuns;)
{
activityRun = enumeratorActivity.Current;
hasMoreActivityRuns = enumeratorActivity.MoveNext();
DataRow dr = dt.NewRow();
dr["ActivityName"] = activityRun.ActivityName ?? (object)DBNull.Value;
dr["RunId"] = parentRunId;
dr["ActivityRunStart"] = activityRun.ActivityRunStart ?? (object)DBNull.Value;
dr["ActivityRunEnd"] = activityRun.ActivityRunEnd ?? (object)DBNull.Value;
dr["ActivityRunId"] = activityRun.ActivityRunId ?? (object)DBNull.Value;
dr["ActivityType"] = activityRun.ActivityType ?? (object)DBNull.Value;
//dr["AdditionalProperties"] = activityRun.AdditionalProperties ?? (object)DBNull.Value;
dr["DurationInMs"] = activityRun.DurationInMs ?? (object)DBNull.Value;
//dr["Error"] = activityRun.Error ?? (object)DBNull.Value;
//dr["Input"] = activityRun.Input ?? (object)DBNull.Value;
//dr["LinkedServiceName"] = activityRun.LinkedServiceName ?? (object)DBNull.Value;
dr["OutPut"] = activityRun.Output ?? (object)DBNull.Value;
dr["PipelineName"] = activityRun.PipelineName ?? (object)DBNull.Value;
dr["PipelineRunId"] = activityRun.PipelineRunId ?? (object)DBNull.Value;
dr["Status"] = activityRun.Status ?? (object)DBNull.Value;
dt.Rows.Add(dr);
if (activityRun.ActivityType == "ExecutePipeline")
{
string _runId = Shared.JsonHelpers.GetStringValueFromJSON(logging, "pipelineRunId", (JObject)activityRun.Output, null, true);
if (!String.IsNullOrEmpty(_runId))
{
QueryActivityRuns(subscriptionId, resourceGroup, factoryName, _runId, parentRunId, filterParameterActivityRuns, logging, ref dt);
}
else
{
logging.LogInformation(String.Format("QueryActivityRuns - RunId Is null for {0} ", (JObject)activityRun.Output));
}
}
}
}
#endregion
}
public ActivityRunsQueryResponse GetActivityRuns(string pipelineName, string runId, RunFilterParameters filterParameters)
{
return(_pipelineRunClient.QueryActivityRuns(pipelineName, runId, filterParameters).Value);
}
public override PipelineErrorDetail GetPipelineRunActivityErrors(PipelineRunRequest request)
{
PipelineRun pipelineRun = _pipelineRunClient.GetPipelineRun
(
request.RunId
);
//Defensive check
PipelineNameCheck(request.PipelineName, pipelineRun.PipelineName);
_logger.LogInformation("Create pipeline Activity Runs query filters.");
RunFilterParameters filterParams = new RunFilterParameters
(
request.ActivityQueryStart,
request.ActivityQueryEnd
);
_logger.LogInformation("Querying SYN pipeline for Activity Runs.");
ActivityRunsQueryResponse queryResponse = _pipelineRunClient.QueryActivityRuns
(
request.PipelineName,
request.RunId,
filterParams
);
//Create initial output content
PipelineErrorDetail output = new PipelineErrorDetail()
{
PipelineName = request.PipelineName,
ActualStatus = pipelineRun.Status,
RunId = request.RunId,
ResponseCount = queryResponse.Value.Count
};
_logger.LogInformation("Pipeline status: " + pipelineRun.Status);
_logger.LogInformation("Activities found in pipeline response: " + queryResponse.Value.Count.ToString());
//Loop over activities in pipeline run
foreach (ActivityRun activity in queryResponse.Value)
{
if (activity.Error == null)
{
continue; //only want errors
}
//Parse error output to customise output
var json = JsonConvert.SerializeObject(activity.Error);
Dictionary <string, object> errorContent = JsonConvert.DeserializeObject <Dictionary <string, object> >(json);
_logger.LogInformation("Activity run id: " + activity.ActivityRunId);
_logger.LogInformation("Activity name: " + activity.ActivityName);
_logger.LogInformation("Activity type: " + activity.ActivityType);
_logger.LogInformation("Error message: " + errorContent["message"].ToString());
output.Errors.Add(new FailedActivity()
{
ActivityRunId = activity.ActivityRunId,
ActivityName = activity.ActivityName,
ActivityType = activity.ActivityType,
ErrorCode = errorContent["errorCode"].ToString(),
ErrorType = errorContent["failureType"].ToString(),
ErrorMessage = errorContent["message"].ToString()
});
}
return(output);
}
/// <summary>
/// Query trigger runs.
/// </summary>
/// <param name='operations'>
/// The operations group for this extension method.
/// </param>
/// <param name='resourceGroupName'>
/// The resource group name.
/// </param>
/// <param name='factoryName'>
/// The factory name.
/// </param>
/// <param name='filterParameters'>
/// Parameters to filter the pipeline run.
/// </param>
/// <param name='cancellationToken'>
/// The cancellation token.
/// </param>
public static async Task <TriggerRunsQueryResponse> QueryByFactoryAsync(this ITriggerRunsOperations operations, string resourceGroupName, string factoryName, RunFilterParameters filterParameters, CancellationToken cancellationToken = default(CancellationToken))
{
using (var _result = await operations.QueryByFactoryWithHttpMessagesAsync(resourceGroupName, factoryName, filterParameters, null, cancellationToken).ConfigureAwait(false))
{
return(_result.Body);
}
}
public IReadOnlyList <TriggerRun> QueryTriggerRunsByWorkspace(RunFilterParameters filterParameters)
{
return(_triggerRunClient.QueryTriggerRunsByWorkspace(filterParameters).Value.Value);
}
public List <ActivityRunsQueryResponse> GetActivityRuns(string pipelineName, string runId, RunFilterParameters filterParameters)
{
List <ActivityRunsQueryResponse> activityRuns = new List <ActivityRunsQueryResponse>();
string ContinuationToken = null;
do
{
var response = _pipelineRunClient.QueryActivityRuns(pipelineName, runId, filterParameters).Value;
ContinuationToken = response.ContinuationToken;
filterParameters.ContinuationToken = ContinuationToken;
activityRuns.Add(response);
}while (!string.IsNullOrWhiteSpace(ContinuationToken));
return(activityRuns);
}
public override PipelineErrorDetail GetPipelineRunActivityErrors(PipelineRunRequest request)
{
PipelineRun pipelineRun = _adfManagementClient.PipelineRuns.Get
(
request.ResourceGroupName,
request.OrchestratorName,
request.RunId
);
//Defensive check
PipelineNameCheck(request.PipelineName, pipelineRun.PipelineName);
_logger.LogInformation("Create pipeline Activity Runs query filters.");
RunFilterParameters filterParams = new RunFilterParameters
(
request.ActivityQueryStart,
request.ActivityQueryEnd
);
_logger.LogInformation("Querying ADF pipeline for Activity Runs.");
ActivityRunsQueryResponse queryResponse = _adfManagementClient.ActivityRuns.QueryByPipelineRun
(
request.ResourceGroupName,
request.OrchestratorName,
request.RunId,
filterParams
);
//Create initial output content
PipelineErrorDetail output = new PipelineErrorDetail()
{
PipelineName = request.PipelineName,
ActualStatus = pipelineRun.Status,
RunId = request.RunId,
ResponseCount = queryResponse.Value.Count
};
_logger.LogInformation("Pipeline status: " + pipelineRun.Status);
_logger.LogInformation("Activities found in pipeline response: " + queryResponse.Value.Count.ToString());
//Loop over activities in pipeline run
foreach (ActivityRun activity in queryResponse.Value)
{
if (activity.Error == null)
{
continue; //only want errors
}
//Parse error output to customise output
dynamic outputBlockInner = JsonConvert.DeserializeObject(activity.Error.ToString());
string errorCode = outputBlockInner?.errorCode;
string errorType = outputBlockInner?.failureType;
string errorMessage = outputBlockInner?.message;
_logger.LogInformation("Activity run id: " + activity.ActivityRunId);
_logger.LogInformation("Activity name: " + activity.ActivityName);
_logger.LogInformation("Activity type: " + activity.ActivityType);
_logger.LogInformation("Error message: " + errorMessage);
output.Errors.Add(new FailedActivity()
{
ActivityRunId = activity.ActivityRunId,
ActivityName = activity.ActivityName,
ActivityType = activity.ActivityType,
ErrorCode = errorCode,
ErrorType = errorType,
ErrorMessage = errorMessage
});
}
return(output);
}
public static async Task <bool> GetLastPipelineStatusByName(DataFactoryManagementClient client, string resourceGroup, string factoryName, string pipelineName, ILogger log)
{
log.LogInformation("C# HTTP trigger function processed a request.");
int daysOfRuns = 30; // int.Parse(Environment.GetEnvironmentVariable("DefaultDaysForPipelineRuns"));
//Get pipeline status
PipelineRun pipelineRuns; //used to find latest pipeline run id
string pipelineStatus = String.Empty;
string runId = String.Empty;
string outputString;
DateTime today = DateTime.Now;
DateTime lastWeek = DateTime.Now.AddDays(-daysOfRuns);
/*
* https://docs.microsoft.com/en-us/rest/api/datafactory/pipelineruns/querybyfactory#runqueryfilteroperand
*/
//Query data factory for pipeline runs
IList <string> pipelineList = new List <string> {
pipelineName
};
IList <RunQueryFilter> moreParams = new List <RunQueryFilter>();
moreParams.Add(new RunQueryFilter
{
Operand = RunQueryFilterOperand.PipelineName,
OperatorProperty = RunQueryFilterOperator.Equals,
Values = pipelineList
});
RunFilterParameters filterParams = new RunFilterParameters(lastWeek, today, null, moreParams, null);
var requiredRuns = client.PipelineRuns.QueryByFactory(resourceGroup, factoryName, filterParams);
var enumerator = requiredRuns.Value.GetEnumerator();
//Get latest run id
for (bool hasMoreRuns = enumerator.MoveNext(); hasMoreRuns;)
{
pipelineRuns = enumerator.Current;
hasMoreRuns = enumerator.MoveNext();
if (!hasMoreRuns && pipelineRuns.PipelineName == pipelineName)
{
//Get status for run id
runId = pipelineRuns.RunId;
pipelineStatus = client.PipelineRuns.Get(resourceGroup, factoryName, runId).Status;
}
}
//Prepare output
outputString = "{ \"PipelineName\": \"" + pipelineName + "\", \"RunIdUsed\": \"" + runId + "\", \"Status\": \"" + pipelineStatus + "\" }";
//JObject json = JObject.Parse(outputString);
if (pipelineStatus.Equals(STATUS_SUCCEEDED))
{
return(true);
}
else
{
return(false);
}
}
public static async Task <IActionResult> Run(
[HttpTrigger(AuthorizationLevel.Anonymous, "get", "post", Route = null)] HttpRequest req,
ILogger log)
{
log.LogInformation("C# HTTP trigger function processed a request.");
string tenantId = req.Query["tenantId"];
string applicationId = req.Query["applicationId"];
string authenticationKey = req.Query["authenticationKey"];
string subscriptionId = req.Query["subscriptionId"];
string resourceGroup = req.Query["resourceGroup"];
string factoryName = req.Query["factoryName"];
string pipelineName = req.Query["pipelineName"];
int daysOfRuns = int.Parse(Environment.GetEnvironmentVariable("DefaultDaysForPipelineRuns"));
string requestBody = await new StreamReader(req.Body).ReadToEndAsync();
dynamic data = JsonConvert.DeserializeObject(requestBody);
tenantId = tenantId ?? data?.tenantId;
applicationId = applicationId ?? data?.applicationId;
authenticationKey = authenticationKey ?? data?.authenticationKey;
subscriptionId = subscriptionId ?? data?.subscriptionId;
resourceGroup = resourceGroup ?? data?.resourceGroup;
factoryName = factoryName ?? data?.factoryName;
pipelineName = pipelineName ?? data?.pipelineName;
if (
tenantId == null ||
applicationId == null ||
authenticationKey == null ||
subscriptionId == null ||
factoryName == null ||
pipelineName == null
)
{
return(new BadRequestObjectResult("Invalid request body, value missing."));
}
// Authenticate and create a data factory management client
var context = new AuthenticationContext("https://login.windows.net/" + tenantId);
ClientCredential cc = new ClientCredential(applicationId, authenticationKey);
AuthenticationResult result = context.AcquireTokenAsync(
"https://management.azure.com/", cc).Result;
ServiceClientCredentials cred = new TokenCredentials(result.AccessToken);
var client = new DataFactoryManagementClient(cred)
{
SubscriptionId = subscriptionId
};
//Get pipeline status
PipelineRun pipelineRuns; //used to find latest pipeline run id
PipelineRun pipelineRun; //used to get the status of the last pipeline
ActivityRunsQueryResponse queryResponse; //used if not successful
string runId = String.Empty;
string errorDetails = String.Empty;
string outputString;
DateTime today = DateTime.Now;
DateTime lastWeek = DateTime.Now.AddDays(-daysOfRuns);
/*
* https://docs.microsoft.com/en-us/rest/api/datafactory/pipelineruns/querybyfactory#runqueryfilteroperand
*/
//Query data factory for pipeline runs
IList <string> pipelineList = new List <string> {
pipelineName
};
IList <RunQueryFilter> moreParams = new List <RunQueryFilter>();
moreParams.Add(new RunQueryFilter
{
Operand = RunQueryFilterOperand.PipelineName,
OperatorProperty = RunQueryFilterOperator.Equals,
Values = pipelineList
});
RunFilterParameters filterParams = new RunFilterParameters(lastWeek, today, null, moreParams, null);
var requiredRuns = client.PipelineRuns.QueryByFactory(resourceGroup, factoryName, filterParams);
var enumerator = requiredRuns.Value.GetEnumerator();
//Get latest run id
for (bool hasMoreRuns = enumerator.MoveNext(); hasMoreRuns;)
{
pipelineRuns = enumerator.Current;
hasMoreRuns = enumerator.MoveNext();
if (!hasMoreRuns && pipelineRuns.PipelineName == pipelineName) //&& just incase, filter above should deal with this
{
//Get run id
runId = pipelineRuns.RunId;
}
}
//Wait for success or fail
while (true)
{
pipelineRun = client.PipelineRuns.Get(resourceGroup, factoryName, runId);
//Console.WriteLine("Status: " + pipelineRun.Status);
if (pipelineRun.Status == "InProgress" || pipelineRun.Status == "Queued")
{
System.Threading.Thread.Sleep(15000);
}
else
{
break;
}
}
//Get error details
if (pipelineRun.Status != "Succeeded")
{
// Check the pipeline if it wasn't successful
RunFilterParameters filterParamsForError = new RunFilterParameters(lastWeek, today);
queryResponse = client.ActivityRuns.QueryByPipelineRun(resourceGroup, factoryName, runId, filterParamsForError);
errorDetails = queryResponse.Value.First().Error.ToString();
}
//Prepare output
outputString = "{ \"PipelineName\": \"" + pipelineName + "\", \"RunIdUsed\": \"" + runId + "\", \"Status\": \"" + pipelineRun.Status + "\" }";
JObject json = JObject.Parse(outputString);
return(pipelineRun.Status == "Succeeded"
? (ActionResult) new OkObjectResult(json)
: new BadRequestObjectResult($"{errorDetails}"));
}
public static async Task <IActionResult> Run(
[HttpTrigger(AuthorizationLevel.Function, "get", "post", Route = null)] HttpRequest req,
ILogger log)
{
log.LogInformation("C# HTTP trigger function processed a request.");
//Get body values
string tenantId = req.Query["tenantId"];
string applicationId = req.Query["applicationId"];
string authenticationKey = req.Query["authenticationKey"];
string subscriptionId = req.Query["subscriptionId"];
string resourceGroup = req.Query["resourceGroup"];
string factoryName = req.Query["factoryName"];
string pipelineName = req.Query["pipelineName"];
int daysOfRuns = int.Parse(Environment.GetEnvironmentVariable("DefaultDaysForPipelineRuns"));
string requestBody = await new StreamReader(req.Body).ReadToEndAsync();
dynamic data = JsonConvert.DeserializeObject(requestBody);
tenantId = tenantId ?? data?.tenantId;
applicationId = applicationId ?? data?.applicationId;
authenticationKey = authenticationKey ?? data?.authenticationKey;
subscriptionId = subscriptionId ?? data?.subscriptionId;
resourceGroup = resourceGroup ?? data?.resourceGroup;
factoryName = factoryName ?? data?.factoryName;
pipelineName = pipelineName ?? data?.pipelineName;
//Check body for values
if (
tenantId == null ||
applicationId == null ||
authenticationKey == null ||
subscriptionId == null ||
factoryName == null ||
pipelineName == null
)
{
return(new BadRequestObjectResult("Invalid request body, value missing."));
}
//Create a data factory management client
var context = new AuthenticationContext("https://login.windows.net/" + tenantId);
ClientCredential cc = new ClientCredential(applicationId, authenticationKey);
AuthenticationResult result = context.AcquireTokenAsync("https://management.azure.com/", cc).Result;
ServiceClientCredentials cred = new TokenCredentials(result.AccessToken);
var client = new DataFactoryManagementClient(cred)
{
SubscriptionId = subscriptionId
};
//Get pipeline status
PipelineRun pipelineRuns; //used to find latest pipeline run id
PipelineRun pipelineRun; //used to get the status of the last pipeline
string pipelineStatus = String.Empty;
string runId = String.Empty;
string outputString;
DateTime today = DateTime.Now;
DateTime lastWeek = DateTime.Now.AddDays(-daysOfRuns);
/*
* https://docs.microsoft.com/en-us/rest/api/datafactory/pipelineruns/querybyfactory#runqueryfilteroperand
*/
//Query data factory for pipeline runs
IList <string> pipelineList = new List <string> {
pipelineName
};
IList <RunQueryFilter> moreParams = new List <RunQueryFilter>();
moreParams.Add(new RunQueryFilter
{
Operand = RunQueryFilterOperand.PipelineName,
OperatorProperty = RunQueryFilterOperator.Equals,
Values = pipelineList
});
RunFilterParameters filterParams = new RunFilterParameters(lastWeek, today, null, moreParams, null);
var requiredRuns = client.PipelineRuns.QueryByFactory(resourceGroup, factoryName, filterParams);
var enumerator = requiredRuns.Value.GetEnumerator();
//Get latest run id
for (bool hasMoreRuns = enumerator.MoveNext(); hasMoreRuns;)
{
pipelineRuns = enumerator.Current;
hasMoreRuns = enumerator.MoveNext();
if (!hasMoreRuns && pipelineRuns.PipelineName == pipelineName)
{
//Get status for run id
runId = pipelineRuns.RunId;
pipelineStatus = client.PipelineRuns.Get(resourceGroup, factoryName, runId).Status;
}
}
//Prepare output
outputString = "{ \"PipelineName\": \"" + pipelineName + "\", \"RunIdUsed\": \"" + runId + "\", \"Status\": \"" + pipelineStatus + "\" }";
JObject json = JObject.Parse(outputString);
return(new OkObjectResult(json));
}
public static async Task <IActionResult> Run(
[HttpTrigger(AuthorizationLevel.Function, "get", "post", Route = null)] HttpRequest req,
ILogger log)
{
log.LogInformation("GetActivityErrors Function triggered by HTTP request.");
#region ParseRequestBody
log.LogInformation("Parsing body from request.");
string requestBody = await new StreamReader(req.Body).ReadToEndAsync();
dynamic inputData = JsonConvert.DeserializeObject(requestBody);
string tenantId = inputData?.tenantId;
string applicationId = inputData?.applicationId;
string authenticationKey = inputData?.authenticationKey;
string subscriptionId = inputData?.subscriptionId;
string resourceGroup = inputData?.resourceGroup;
string factoryName = inputData?.factoryName;
string pipelineName = inputData?.pipelineName;
string runId = inputData?.runId;
//Check body for values
if (
tenantId == null ||
applicationId == null ||
authenticationKey == null ||
subscriptionId == null ||
resourceGroup == null ||
factoryName == null ||
pipelineName == null ||
runId == null
)
{
log.LogInformation("Invalid body.");
return(new BadRequestObjectResult("Invalid request body, value(s) missing."));
}
#endregion
//Query and output support variables
int daysOfRuns = 7; //max duration for mandatory RunFilterParameters
DateTime today = DateTime.Now;
DateTime lastWeek = DateTime.Now.AddDays(-daysOfRuns);
dynamic outputBlock = new JObject();
dynamic outputBlockInner;
//Create a data factory management client
log.LogInformation("Creating ADF connectivity client.");
using (var client = DataFactoryClient.CreateDataFactoryClient(tenantId, applicationId, authenticationKey, subscriptionId))
{
#region SetPipelineRunDetails
//Get pipeline details
PipelineRun pipelineRun;
pipelineRun = client.PipelineRuns.Get(resourceGroup, factoryName, runId);
log.LogInformation("Querying ADF pipeline for Activity Runs.");
RunFilterParameters filterParams = new RunFilterParameters(lastWeek, today);
ActivityRunsQueryResponse queryResponse = client.ActivityRuns.QueryByPipelineRun(resourceGroup, factoryName, runId, filterParams);
//Create initial output content
outputBlock.PipelineName = pipelineName;
outputBlock.PipelineStatus = pipelineRun.Status;
outputBlock.RunId = runId;
outputBlock.ResponseCount = queryResponse.Value.Count;
outputBlock.ResponseErrorCount = 0;
outputBlock.Errors = new JArray();
JObject errorDetails;
#endregion
log.LogInformation("Pipeline status: " + pipelineRun.Status);
log.LogInformation("Activities found in pipeline response: " + queryResponse.Value.Count.ToString());
#region GetActivityDetails
//Loop over activities in pipeline run
foreach (var activity in queryResponse.Value)
{
if (string.IsNullOrEmpty(activity.Error.ToString()))
{
continue; //just incase
}
//Parse error output to customise output
outputBlockInner = JsonConvert.DeserializeObject(activity.Error.ToString());
string errorCode = outputBlockInner?.errorCode;
string errorType = outputBlockInner?.failureType;
string errorMessage = outputBlockInner?.message;
//Get output details
if (!string.IsNullOrEmpty(errorCode))
{
log.LogInformation("Activity run id: " + activity.ActivityRunId);
log.LogInformation("Activity name: " + activity.ActivityName);
log.LogInformation("Activity type: " + activity.ActivityType);
log.LogInformation("Error message: " + errorMessage);
outputBlock.ResponseErrorCount += 1;
//Construct custom error information block
errorDetails = JObject.Parse("{ \"ActivityRunId\": \"" + activity.ActivityRunId +
"\", \"ActivityName\": \"" + activity.ActivityName +
"\", \"ActivityType\": \"" + activity.ActivityType +
"\", \"ErrorCode\": \"" + errorCode +
"\", \"ErrorType\": \"" + errorType +
"\", \"ErrorMessage\": \"" + errorMessage +
"\" }");
outputBlock.Errors.Add(errorDetails);
}
}
#endregion
}
log.LogInformation("GetActivityErrors Function complete.");
return(new OkObjectResult(outputBlock));
}
static void Main(string[] args)
{
// Set variables
string tenantID = "<your tenant ID>";
string applicationId = "<your application ID>";
string authenticationKey = "<your authentication key for the application>";
string subscriptionId = "<your subscription ID to create the factory>";
string resourceGroup = "<your resource group to create the factory>";
string region = "East US";
string dataFactoryName = "<specify the name of a data factory to create. It must be globally unique.>";
// Specify the source Azure Blob information
string storageAccount = "<your storage account name to copy data>";
string storageKey = "<your storage account key>";
string inputBlobPath = "adfv2tutorial/";
string inputBlobName = "inputEmp.txt";
// Specify the sink Azure SQL Database information
string azureSqlConnString = "Server=tcp:<your server name>.database.windows.net,1433;Database=<your database name>;User ID=<your username>@<your server name>;Password=<your password>;Trusted_Connection=False;Encrypt=True;Connection Timeout=30";
string azureSqlTableName = "dbo.emp";
string storageLinkedServiceName = "AzureStorageLinkedService";
string sqlDbLinkedServiceName = "AzureSqlDbLinkedService";
string blobDatasetName = "BlobDataset";
string sqlDatasetName = "SqlDataset";
string pipelineName = "Adfv2TutorialBlobToSqlCopy";
// Authenticate and create a data factory management client
var context = new AuthenticationContext("https://login.windows.net/" + tenantID);
ClientCredential cc = new ClientCredential(applicationId, authenticationKey);
AuthenticationResult result = context.AcquireTokenAsync("https://management.azure.com/", cc).Result;
ServiceClientCredentials cred = new TokenCredentials(result.AccessToken);
var client = new DataFactoryManagementClient(cred)
{
SubscriptionId = subscriptionId
};
// Create a data factory
Console.WriteLine("Creating a data factory " + dataFactoryName + "...");
Factory dataFactory = new Factory
{
Location = region,
Identity = new FactoryIdentity()
};
client.Factories.CreateOrUpdate(resourceGroup, dataFactoryName, dataFactory);
Console.WriteLine(SafeJsonConvert.SerializeObject(dataFactory, client.SerializationSettings));
while (client.Factories.Get(resourceGroup, dataFactoryName).ProvisioningState == "PendingCreation")
{
System.Threading.Thread.Sleep(1000);
}
// Create an Azure Storage linked service
Console.WriteLine("Creating linked service " + storageLinkedServiceName + "...");
LinkedServiceResource storageLinkedService = new LinkedServiceResource(
new AzureStorageLinkedService
{
ConnectionString = new SecureString("DefaultEndpointsProtocol=https;AccountName=" + storageAccount + ";AccountKey=" + storageKey)
}
);
client.LinkedServices.CreateOrUpdate(resourceGroup, dataFactoryName, storageLinkedServiceName, storageLinkedService);
Console.WriteLine(SafeJsonConvert.SerializeObject(storageLinkedService, client.SerializationSettings));
// Create an Azure SQL Database linked service
Console.WriteLine("Creating linked service " + sqlDbLinkedServiceName + "...");
LinkedServiceResource sqlDbLinkedService = new LinkedServiceResource(
new AzureSqlDatabaseLinkedService
{
ConnectionString = new SecureString(azureSqlConnString)
}
);
client.LinkedServices.CreateOrUpdate(resourceGroup, dataFactoryName, sqlDbLinkedServiceName, sqlDbLinkedService);
Console.WriteLine(SafeJsonConvert.SerializeObject(sqlDbLinkedService, client.SerializationSettings));
// Create an Azure Blob dataset
Console.WriteLine("Creating dataset " + blobDatasetName + "...");
DatasetResource blobDataset = new DatasetResource(
new AzureBlobDataset
{
LinkedServiceName = new LinkedServiceReference
{
ReferenceName = storageLinkedServiceName
},
FolderPath = inputBlobPath,
FileName = inputBlobName,
Format = new TextFormat {
ColumnDelimiter = "|"
},
Structure = new List <DatasetDataElement>
{
new DatasetDataElement
{
Name = "FirstName",
Type = "String"
},
new DatasetDataElement
{
Name = "LastName",
Type = "String"
}
}
}
);
client.Datasets.CreateOrUpdate(resourceGroup, dataFactoryName, blobDatasetName, blobDataset);
Console.WriteLine(SafeJsonConvert.SerializeObject(blobDataset, client.SerializationSettings));
// Create an Azure SQL Database dataset
Console.WriteLine("Creating dataset " + sqlDatasetName + "...");
DatasetResource sqlDataset = new DatasetResource(
new AzureSqlTableDataset
{
LinkedServiceName = new LinkedServiceReference
{
ReferenceName = sqlDbLinkedServiceName
},
TableName = azureSqlTableName
}
);
client.Datasets.CreateOrUpdate(resourceGroup, dataFactoryName, sqlDatasetName, sqlDataset);
Console.WriteLine(SafeJsonConvert.SerializeObject(sqlDataset, client.SerializationSettings));
// Create a pipeline with copy activity
Console.WriteLine("Creating pipeline " + pipelineName + "...");
PipelineResource pipeline = new PipelineResource
{
Activities = new List <Activity>
{
new CopyActivity
{
Name = "CopyFromBlobToSQL",
Inputs = new List <DatasetReference>
{
new DatasetReference()
{
ReferenceName = blobDatasetName
}
},
Outputs = new List <DatasetReference>
{
new DatasetReference
{
ReferenceName = sqlDatasetName
}
},
Source = new BlobSource {
},
Sink = new SqlSink {
}
}
}
};
client.Pipelines.CreateOrUpdate(resourceGroup, dataFactoryName, pipelineName, pipeline);
Console.WriteLine(SafeJsonConvert.SerializeObject(pipeline, client.SerializationSettings));
// Create a pipeline run
Console.WriteLine("Creating pipeline run...");
CreateRunResponse runResponse = client.Pipelines.CreateRunWithHttpMessagesAsync(resourceGroup, dataFactoryName, pipelineName).Result.Body;
Console.WriteLine("Pipeline run ID: " + runResponse.RunId);
// Monitor the pipeline run
Console.WriteLine("Checking pipeline run status...");
PipelineRun pipelineRun;
while (true)
{
pipelineRun = client.PipelineRuns.Get(resourceGroup, dataFactoryName, runResponse.RunId);
Console.WriteLine("Status: " + pipelineRun.Status);
if (pipelineRun.Status == "InProgress")
{
System.Threading.Thread.Sleep(15000);
}
else
{
break;
}
}
// Check the copy activity run details
Console.WriteLine("Checking copy activity run details...");
var runFilterParameters = new RunFilterParameters(DateTime.UtcNow.AddMinutes(-10), DateTime.UtcNow.AddMinutes(10));
var activityRuns = client.ActivityRuns.QueryByPipelineRun(
resourceGroup, dataFactoryName, runResponse.RunId, runFilterParameters);
if (pipelineRun.Status == "Succeeded")
{
Console.WriteLine(activityRuns.Value.First().Output);
}
else
{
Console.WriteLine(activityRuns.Value.First().Error);
}
Console.WriteLine("\nPress any key to exit...");
Console.ReadKey();
}
/// <summary>
/// Query pipeline runs in the factory based on input filter conditions.
/// </summary>
/// <param name='resourceGroupName'>
/// The resource group name.
/// </param>
/// <param name='factoryName'>
/// The factory name.
/// </param>
/// <param name='filterParameters'>
/// Parameters to filter the pipeline run.
/// </param>
/// <param name='customHeaders'>
/// Headers that will be added to request.
/// </param>
/// <param name='cancellationToken'>
/// The cancellation token.
/// </param>
/// <exception cref="CloudException">
/// Thrown when the operation returned an invalid status code
/// </exception>
/// <exception cref="SerializationException">
/// Thrown when unable to deserialize the response
/// </exception>
/// <exception cref="ValidationException">
/// Thrown when a required parameter is null
/// </exception>
/// <exception cref="System.ArgumentNullException">
/// Thrown when a required parameter is null
/// </exception>
/// <return>
/// A response object containing the response body and response headers.
/// </return>
public async Task <AzureOperationResponse <PipelineRunsQueryResponse> > QueryByFactoryWithHttpMessagesAsync(string resourceGroupName, string factoryName, RunFilterParameters filterParameters, Dictionary <string, List <string> > customHeaders = null, CancellationToken cancellationToken = default(CancellationToken))
{
if (Client.SubscriptionId == null)
{
throw new ValidationException(ValidationRules.CannotBeNull, "this.Client.SubscriptionId");
}
if (resourceGroupName == null)
{
throw new ValidationException(ValidationRules.CannotBeNull, "resourceGroupName");
}
if (resourceGroupName != null)
{
if (resourceGroupName.Length > 90)
{
throw new ValidationException(ValidationRules.MaxLength, "resourceGroupName", 90);
}
if (resourceGroupName.Length < 1)
{
throw new ValidationException(ValidationRules.MinLength, "resourceGroupName", 1);
}
if (!System.Text.RegularExpressions.Regex.IsMatch(resourceGroupName, "^[-\\w\\._\\(\\)]+$"))
{
throw new ValidationException(ValidationRules.Pattern, "resourceGroupName", "^[-\\w\\._\\(\\)]+$");
}
}
if (factoryName == null)
{
throw new ValidationException(ValidationRules.CannotBeNull, "factoryName");
}
if (factoryName != null)
{
if (factoryName.Length > 63)
{
throw new ValidationException(ValidationRules.MaxLength, "factoryName", 63);
}
if (factoryName.Length < 3)
{
throw new ValidationException(ValidationRules.MinLength, "factoryName", 3);
}
if (!System.Text.RegularExpressions.Regex.IsMatch(factoryName, "^[A-Za-z0-9]+(?:-[A-Za-z0-9]+)*$"))
{
throw new ValidationException(ValidationRules.Pattern, "factoryName", "^[A-Za-z0-9]+(?:-[A-Za-z0-9]+)*$");
}
}
if (Client.ApiVersion == null)
{
throw new ValidationException(ValidationRules.CannotBeNull, "this.Client.ApiVersion");
}
if (filterParameters == null)
{
throw new ValidationException(ValidationRules.CannotBeNull, "filterParameters");
}
if (filterParameters != null)
{
filterParameters.Validate();
}
// Tracing
bool _shouldTrace = ServiceClientTracing.IsEnabled;
string _invocationId = null;
if (_shouldTrace)
{
_invocationId = ServiceClientTracing.NextInvocationId.ToString();
Dictionary <string, object> tracingParameters = new Dictionary <string, object>();
tracingParameters.Add("resourceGroupName", resourceGroupName);
tracingParameters.Add("factoryName", factoryName);
tracingParameters.Add("filterParameters", filterParameters);
tracingParameters.Add("cancellationToken", cancellationToken);
ServiceClientTracing.Enter(_invocationId, this, "QueryByFactory", tracingParameters);
}
// Construct URL
var _baseUrl = Client.BaseUri.AbsoluteUri;
var _url = new System.Uri(new System.Uri(_baseUrl + (_baseUrl.EndsWith("/") ? "" : "/")), "subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.DataFactory/factories/{factoryName}/queryPipelineRuns").ToString();
_url = _url.Replace("{subscriptionId}", System.Uri.EscapeDataString(Client.SubscriptionId));
_url = _url.Replace("{resourceGroupName}", System.Uri.EscapeDataString(resourceGroupName));
_url = _url.Replace("{factoryName}", System.Uri.EscapeDataString(factoryName));
List <string> _queryParameters = new List <string>();
if (Client.ApiVersion != null)
{
_queryParameters.Add(string.Format("api-version={0}", System.Uri.EscapeDataString(Client.ApiVersion)));
}
if (_queryParameters.Count > 0)
{
_url += (_url.Contains("?") ? "&" : "?") + string.Join("&", _queryParameters);
}
// Create HTTP transport objects
var _httpRequest = new HttpRequestMessage();
HttpResponseMessage _httpResponse = null;
_httpRequest.Method = new HttpMethod("POST");
_httpRequest.RequestUri = new System.Uri(_url);
// Set Headers
if (Client.GenerateClientRequestId != null && Client.GenerateClientRequestId.Value)
{
_httpRequest.Headers.TryAddWithoutValidation("x-ms-client-request-id", System.Guid.NewGuid().ToString());
}
if (Client.AcceptLanguage != null)
{
if (_httpRequest.Headers.Contains("accept-language"))
{
_httpRequest.Headers.Remove("accept-language");
}
_httpRequest.Headers.TryAddWithoutValidation("accept-language", Client.AcceptLanguage);
}
if (customHeaders != null)
{
foreach (var _header in customHeaders)
{
if (_httpRequest.Headers.Contains(_header.Key))
{
_httpRequest.Headers.Remove(_header.Key);
}
_httpRequest.Headers.TryAddWithoutValidation(_header.Key, _header.Value);
}
}
// Serialize Request
string _requestContent = null;
if (filterParameters != null)
{
_requestContent = Rest.Serialization.SafeJsonConvert.SerializeObject(filterParameters, Client.SerializationSettings);
_httpRequest.Content = new StringContent(_requestContent, System.Text.Encoding.UTF8);
_httpRequest.Content.Headers.ContentType = System.Net.Http.Headers.MediaTypeHeaderValue.Parse("application/json; charset=utf-8");
}
// Set Credentials
if (Client.Credentials != null)
{
cancellationToken.ThrowIfCancellationRequested();
await Client.Credentials.ProcessHttpRequestAsync(_httpRequest, cancellationToken).ConfigureAwait(false);
}
// Send Request
if (_shouldTrace)
{
ServiceClientTracing.SendRequest(_invocationId, _httpRequest);
}
cancellationToken.ThrowIfCancellationRequested();
_httpResponse = await Client.HttpClient.SendAsync(_httpRequest, cancellationToken).ConfigureAwait(false);
if (_shouldTrace)
{
ServiceClientTracing.ReceiveResponse(_invocationId, _httpResponse);
}
HttpStatusCode _statusCode = _httpResponse.StatusCode;
cancellationToken.ThrowIfCancellationRequested();
string _responseContent = null;
if ((int)_statusCode != 200)
{
var ex = new CloudException(string.Format("Operation returned an invalid status code '{0}'", _statusCode));
try
{
_responseContent = await _httpResponse.Content.ReadAsStringAsync().ConfigureAwait(false);
CloudError _errorBody = Rest.Serialization.SafeJsonConvert.DeserializeObject <CloudError>(_responseContent, Client.DeserializationSettings);
if (_errorBody != null)
{
ex = new CloudException(_errorBody.Message);
ex.Body = _errorBody;
}
}
catch (JsonException)
{
// Ignore the exception
}
ex.Request = new HttpRequestMessageWrapper(_httpRequest, _requestContent);
ex.Response = new HttpResponseMessageWrapper(_httpResponse, _responseContent);
if (_httpResponse.Headers.Contains("x-ms-request-id"))
{
ex.RequestId = _httpResponse.Headers.GetValues("x-ms-request-id").FirstOrDefault();
}
if (_shouldTrace)
{
ServiceClientTracing.Error(_invocationId, ex);
}
_httpRequest.Dispose();
if (_httpResponse != null)
{
_httpResponse.Dispose();
}
throw ex;
}
// Create Result
var _result = new AzureOperationResponse <PipelineRunsQueryResponse>();
_result.Request = _httpRequest;
_result.Response = _httpResponse;
if (_httpResponse.Headers.Contains("x-ms-request-id"))
{
_result.RequestId = _httpResponse.Headers.GetValues("x-ms-request-id").FirstOrDefault();
}
// Deserialize Response
if ((int)_statusCode == 200)
{
_responseContent = await _httpResponse.Content.ReadAsStringAsync().ConfigureAwait(false);
try
{
_result.Body = Rest.Serialization.SafeJsonConvert.DeserializeObject <PipelineRunsQueryResponse>(_responseContent, Client.DeserializationSettings);
}
catch (JsonException ex)
{
_httpRequest.Dispose();
if (_httpResponse != null)
{
_httpResponse.Dispose();
}
throw new SerializationException("Unable to deserialize the response.", _responseContent, ex);
}
}
if (_shouldTrace)
{
ServiceClientTracing.Exit(_invocationId, _result);
}
return(_result);
}
public virtual List <PSActivityRun> ListActivityRuns(ActivityRunFilterOptions activityRunFilter)
{
List <PSActivityRun> activityRuns = new List <PSActivityRun>();
var runFilters = new RunFilterParameters()
{
LastUpdatedAfter = activityRunFilter.RunStartedAfter,
LastUpdatedBefore = activityRunFilter.RunStartedBefore,
Filters = new List <RunQueryFilter>(),
OrderBy = new List <RunQueryOrderBy>(),
};
if (activityRunFilter.ActivityName != null)
{
runFilters.Filters.Add(
new RunQueryFilter()
{
Operand = RunQueryFilterOperand.ActivityName,
OperatorProperty = RunQueryFilterOperator.Equals,
Values = new List <string>()
{
activityRunFilter.ActivityName
}
});
}
if (activityRunFilter.Status != null)
{
runFilters.Filters.Add(
new RunQueryFilter()
{
Operand = RunQueryFilterOperand.Status,
OperatorProperty = RunQueryFilterOperator.Equals,
Values = new List <string>()
{
activityRunFilter.Status
}
});
}
ActivityRunsQueryResponse response = this.DataFactoryManagementClient.ActivityRuns.QueryByPipelineRun(
activityRunFilter.ResourceGroupName,
activityRunFilter.DataFactoryName,
activityRunFilter.PipelineRunId,
runFilters);
activityRuns.AddRange(response.Value.Select(ar =>
new PSActivityRun(ar, activityRunFilter.ResourceGroupName, activityRunFilter.DataFactoryName)));
string continuationToken = response.ContinuationToken;
while (!string.IsNullOrWhiteSpace(continuationToken))
{
runFilters.ContinuationToken = continuationToken;
response = this.DataFactoryManagementClient.ActivityRuns.QueryByPipelineRun(activityRunFilter.ResourceGroupName,
activityRunFilter.DataFactoryName, activityRunFilter.PipelineRunId, runFilters);
activityRuns.AddRange(response.Value.Select(ar =>
new PSActivityRun(ar, activityRunFilter.ResourceGroupName, activityRunFilter.DataFactoryName)));
continuationToken = response.ContinuationToken;
}
return(activityRuns);
}
/// <summary>
/// Query trigger runs.
/// </summary>
/// <param name='operations'>
/// The operations group for this extension method.
/// </param>
/// <param name='resourceGroupName'>
/// The resource group name.
/// </param>
/// <param name='factoryName'>
/// The factory name.
/// </param>
/// <param name='filterParameters'>
/// Parameters to filter the pipeline run.
/// </param>
public static TriggerRunsQueryResponse QueryByFactory(this ITriggerRunsOperations operations, string resourceGroupName, string factoryName, RunFilterParameters filterParameters)
{
return(operations.QueryByFactoryAsync(resourceGroupName, factoryName, filterParameters).GetAwaiter().GetResult());
}
public virtual List <PSPipelineRun> ListPipelineRuns(PipelineRunFilterOptions pipelineRunFilter)
{
var pipelineRuns = new List <PSPipelineRun>();
var runFilters = new RunFilterParameters()
{
LastUpdatedAfter = pipelineRunFilter.LastUpdatedAfter,
LastUpdatedBefore = pipelineRunFilter.LastUpdatedBefore,
Filters = new List <RunQueryFilter>(),
OrderBy = new List <RunQueryOrderBy>(),
};
if (pipelineRunFilter.PipelineName != null)
{
runFilters.Filters.Add(
new RunQueryFilter()
{
Operand = RunQueryFilterOperand.PipelineName,
OperatorProperty = RunQueryFilterOperator.Equals,
Values = new List <string>()
{
pipelineRunFilter.PipelineName
}
});
}
else if (pipelineRunFilter.Filters != null)
{
runFilters.Filters = pipelineRunFilter.Filters;
}
if (pipelineRunFilter.OrderBy != null)
{
runFilters.OrderBy = pipelineRunFilter.OrderBy;
}
else
{
runFilters.OrderBy.Add(
new RunQueryOrderBy()
{
Order = RunQueryOrder.DESC,
OrderBy = RunQueryOrderByField.RunEnd
});
}
PipelineRunsQueryResponse response = this.DataFactoryManagementClient.PipelineRuns.QueryByFactory(pipelineRunFilter.ResourceGroupName, pipelineRunFilter.DataFactoryName, runFilters);
pipelineRuns.AddRange(response.Value.Select(pr =>
new PSPipelineRun(pr, pipelineRunFilter.ResourceGroupName, pipelineRunFilter.DataFactoryName)));
string continuationToken = response.ContinuationToken;
while (!string.IsNullOrWhiteSpace(continuationToken))
{
runFilters.ContinuationToken = continuationToken;
response = this.DataFactoryManagementClient.PipelineRuns.QueryByFactory(pipelineRunFilter.ResourceGroupName, pipelineRunFilter.DataFactoryName, runFilters);
pipelineRuns.AddRange(response.Value.Select(pr =>
new PSPipelineRun(pr, pipelineRunFilter.ResourceGroupName, pipelineRunFilter.DataFactoryName)));
continuationToken = response.ContinuationToken;
}
return(pipelineRuns);
}
public async Task <Response <TriggerRunsQueryResponse> > QueryTriggerRunsByWorkspaceAsync(RunFilterParameters filterParameters, CancellationToken cancellationToken = default)
{
if (filterParameters == null)
{
throw new ArgumentNullException(nameof(filterParameters));
}
using var message = CreateQueryTriggerRunsByWorkspaceRequest(filterParameters);
await _pipeline.SendAsync(message, cancellationToken).ConfigureAwait(false);
switch (message.Response.Status)
{
case 200:
{
TriggerRunsQueryResponse value = default;
using var document = await JsonDocument.ParseAsync(message.Response.ContentStream, default, cancellationToken).ConfigureAwait(false);
value = TriggerRunsQueryResponse.DeserializeTriggerRunsQueryResponse(document.RootElement);
return(Response.FromValue(value, message.Response));
}
//static void Main(string[] args)
static async System.Threading.Tasks.Task Main(string[] args)
{
// Set variables
string tenantID = "microsoft.onmicrosoft.com";
string User = "******";
string applicationId = "<your application ID>";
string authenticationKey = "<your authentication key for the application>";
string subscriptionId = "b67993e8-0937-4812-af89-520a495da302";
string resourceGroup = "vchmeha";
string region = "West Central US";
string dataFactoryName = "ADFv2HaHu";
string storageAccount = "fromblob";
string storageKey = "gREZ3TbADUcfWJ+ryfxOH7FKrlCKobMHrpS7Z2PPS/gQIsH02LHAGedlqYVpxm/PeWHDJsU/wkBmNz1Q20Rwyg==";
// specify the container and input folder from which all files
// need to be copied to the output folder.
string inputBlobPath = "adftutorial/input";
//specify the contains and output folder where the files are copied
string outputBlobPath = "adftutorial/output";
// name of the Azure Storage linked service, blob dataset, and the pipeline
string storageLinkedServiceName = "AzureStorageLinkedService";
string blobDatasetName = "BlobDataset";
string pipelineName = "Adfv2QuickStartPipeline";
// Authenticate and create a data factory management client
var azureServiceTokenProvider = new AzureServiceTokenProvider();
var token = await azureServiceTokenProvider.GetAccessTokenAsync("https://management.azure.com/");
var cred = new TokenCredentials(token);
var client = new DataFactoryManagementClient(cred)
{
SubscriptionId = subscriptionId
};
// Create a data factory
Console.WriteLine("Creating data factory " + dataFactoryName + "...");
Factory dataFactory = new Factory
{
Location = region,
Identity = new FactoryIdentity()
};
client.Factories.CreateOrUpdate(resourceGroup, dataFactoryName, dataFactory);
Console.WriteLine(
SafeJsonConvert.SerializeObject(dataFactory, client.SerializationSettings));
while (client.Factories.Get(resourceGroup, dataFactoryName).ProvisioningState ==
"PendingCreation")
{
System.Threading.Thread.Sleep(1000);
}
// Create an Azure Storage linked service
Console.WriteLine("Creating linked service " + storageLinkedServiceName + "...");
LinkedServiceResource storageLinkedService = new LinkedServiceResource(
new AzureStorageLinkedService
{
ConnectionString = new SecureString(
"DefaultEndpointsProtocol=https;AccountName=" + storageAccount +
";AccountKey=" + storageKey)
}
);
client.LinkedServices.CreateOrUpdate(
resourceGroup, dataFactoryName, storageLinkedServiceName, storageLinkedService);
Console.WriteLine(SafeJsonConvert.SerializeObject(
storageLinkedService, client.SerializationSettings));
// Create an Azure Blob dataset
Console.WriteLine("Creating dataset " + blobDatasetName + "...");
DatasetResource blobDataset = new DatasetResource(
new AzureBlobDataset
{
LinkedServiceName = new LinkedServiceReference
{
ReferenceName = storageLinkedServiceName
},
FolderPath = new Expression {
Value = "@{dataset().path}"
},
Parameters = new Dictionary <string, ParameterSpecification>
{
{ "path", new ParameterSpecification {
Type = ParameterType.String
} }
}
}
);
client.Datasets.CreateOrUpdate(
resourceGroup, dataFactoryName, blobDatasetName, blobDataset);
Console.WriteLine(
SafeJsonConvert.SerializeObject(blobDataset, client.SerializationSettings));
// Create a pipeline with a copy activity
Console.WriteLine("Creating pipeline " + pipelineName + "...");
PipelineResource pipeline = new PipelineResource
{
Parameters = new Dictionary <string, ParameterSpecification>
{
{ "inputPath", new ParameterSpecification {
Type = ParameterType.String
} },
{ "outputPath", new ParameterSpecification {
Type = ParameterType.String
} }
},
Activities = new List <Activity>
{
new CopyActivity
{
Name = "CopyFromBlobToBlob",
Inputs = new List <DatasetReference>
{
new DatasetReference()
{
ReferenceName = blobDatasetName,
Parameters = new Dictionary <string, object>
{
{ "path", "@pipeline().parameters.inputPath" }
}
}
},
Outputs = new List <DatasetReference>
{
new DatasetReference
{
ReferenceName = blobDatasetName,
Parameters = new Dictionary <string, object>
{
{ "path", "@pipeline().parameters.outputPath" }
}
}
},
Source = new BlobSource {
},
Sink = new BlobSink {
}
}
}
};
client.Pipelines.CreateOrUpdate(resourceGroup, dataFactoryName, pipelineName, pipeline);
Console.WriteLine(SafeJsonConvert.SerializeObject(pipeline, client.SerializationSettings));
// Create a pipeline run
Console.WriteLine("Creating pipeline run...");
Dictionary <string, object> parameters = new Dictionary <string, object>
{
{ "inputPath", inputBlobPath },
{ "outputPath", outputBlobPath }
};
CreateRunResponse runResponse = client.Pipelines.CreateRunWithHttpMessagesAsync(
resourceGroup, dataFactoryName, pipelineName, parameters: parameters
).Result.Body;
Console.WriteLine("Pipeline run ID: " + runResponse.RunId);
// Monitor the pipeline run
Console.WriteLine("Checking pipeline run status...");
PipelineRun pipelineRun;
while (true)
{
pipelineRun = client.PipelineRuns.Get(
resourceGroup, dataFactoryName, runResponse.RunId);
Console.WriteLine("Status: " + pipelineRun.Status);
if (pipelineRun.Status == "InProgress" || pipelineRun.Status == "Queued")
{
System.Threading.Thread.Sleep(15000);
}
else
{
break;
}
}
// Check the copy activity run details
Console.WriteLine("Checking copy activity run details...");
RunFilterParameters filterParams = new RunFilterParameters(
DateTime.UtcNow.AddMinutes(-10), DateTime.UtcNow.AddMinutes(10));
ActivityRunsQueryResponse queryResponse = client.ActivityRuns.QueryByPipelineRun(
resourceGroup, dataFactoryName, runResponse.RunId, filterParams);
if (pipelineRun.Status == "Succeeded")
{
Console.WriteLine(queryResponse.Value.First().Output);
}
else
{
Console.WriteLine(queryResponse.Value.First().Error);
}
Console.WriteLine("\nPress any key to exit...");
Console.ReadKey();
}
public IReadOnlyList <PipelineRun> QueryPipelineRunsByWorkspace(RunFilterParameters filterParameters)
{
return(_pipelineRunClient.QueryPipelineRunsByWorkspace(filterParameters).Value.Value);
}
public virtual async Task <Response <TriggerRunsQueryResponse> > QueryTriggerRunsByWorkspaceAsync(RunFilterParameters filterParameters, CancellationToken cancellationToken = default)
{
using var scope = _clientDiagnostics.CreateScope("TriggerRunClient.QueryTriggerRunsByWorkspace");
scope.Start();
try
{
return(await RestClient.QueryTriggerRunsByWorkspaceAsync(filterParameters, cancellationToken).ConfigureAwait(false));
}
catch (Exception e)
{
scope.Failed(e);
throw;
}
}
public virtual Response <ActivityRunsQueryResponse> QueryActivityRuns(string pipelineName, string runId, RunFilterParameters filterParameters, CancellationToken cancellationToken = default)
{
using var scope = _clientDiagnostics.CreateScope("PipelineRunClient.QueryActivityRuns");
scope.Start();
try
{
return(RestClient.QueryActivityRuns(pipelineName, runId, filterParameters, cancellationToken));
}
catch (Exception e)
{
scope.Failed(e);
throw;
}
}
public static async Task Run([TimerTrigger("0 */5 * * * *")] TimerInfo myTimer, ILogger log)
{
log.LogInformation($"C# Timer trigger function executed at: {DateTime.Now}");
// This section needs to be updated before running the Project
#region Get Config
var clientId = "Client Id for the Service Principal registered";
var clientSecret = "Client Secret for the Service Principal registered";
var tenantId = "Tenant Id for the azure tenant";
var subscriptionId = "Subscription Id for the azure subscription";
var resourceGroupName = "resourceGroupName for the factory to be queried";
var factoryName = "Name of the factory to be queried";
#endregion
//Generate authentication token with the client Id and secret of the registered service Principal
var context = new AuthenticationContext("https://login.windows.net/" + tenantId);
ClientCredential cc = new ClientCredential(clientId, clientSecret);
AuthenticationResult authResult = context.AcquireTokenAsync(
"https://management.azure.com/", cc).Result;
ServiceClientCredentials cred = new TokenCredentials(authResult.AccessToken);
// Create Data Factory Client and use the auth token
var client = new Microsoft.Azure.Management.DataFactory.DataFactoryManagementClient(cred)
{
SubscriptionId = subscriptionId
};
List <RunQueryFilter> listQuery = new List <RunQueryFilter>();
#region FilterClause and Run Parameters
RunQueryFilter queryFilter = new RunQueryFilter()
{
Operand = "Status",
OperatorProperty = "Equals",
Values = new List <string> {
"InProgress"
}
};
listQuery.Add(queryFilter);
RunFilterParameters runFilterParameters = new RunFilterParameters()
{
LastUpdatedAfter = DateTime.UtcNow.AddHours(-10),
LastUpdatedBefore = DateTime.UtcNow,
Filters = listQuery
};
#endregion
//Query Data factory
var queryResult = await client.PipelineRuns.QueryByFactoryAsync(resourceGroupName, factoryName, runFilterParameters);
//Serialize the incoming data in a model and store in DB if required
List <FullDataADFMonitoringModel> modelList = new List <FullDataADFMonitoringModel>();
var serializedResult = JsonConvert.SerializeObject(queryResult?.Value);
modelList = JsonConvert.DeserializeObject <List <FullDataADFMonitoringModel> >(serializedResult);
foreach (var item in modelList)
{
item.FactoryName = factoryName;
}
using (System.Data.SqlClient.SqlConnection connectionAzureDB = new SqlConnection("conn string"))
{
var serilaizedModel = JsonConvert.SerializeObject(modelList);
var values = new { AdfPipeLineMonitoringData = "data model to be inserted" };
var sqlResults = await connectionAzureDB.QueryAsync("[Internal].[usp_AzureADFPipeLinesExecutionData]", values, commandTimeout : 0, commandType : CommandType.StoredProcedure);
connectionAzureDB.Close();
}
// Send Mail using sendgrid to concerned team if a pipeline exceeds configured threshold
foreach (var item in modelList)
{
if (item.Status.ToLower() == "inprogress")
{
var timeDiff = DateTime.Now - Convert.ToDateTime(item.RunStart);
// Can be configured based on each pipeline in the factory
var threshold = 10;
if (threshold < timeDiff.Hours)
{
var sendMailUtility = new FullDataSendMailUtility();
await sendMailUtility.SendMailAsync(item, timeDiff, "sendGridApiKey", threshold);
}
}
}
}
