/// <summary>
/// Checks for long running pipelines and updates their status in the database
/// </summary>
/// <param name="logging"></param>
/// <returns></returns>
public static short CheckLongRunningPipelines(Logging logging)
{
dynamic _ActivePipelines = ActivePipelines.GetLongRunningPipelines(logging);
short RunningPipelines = 0;
short FinishedPipelines = 0;
DataTable dt = new DataTable();
dt.Columns.Add(new DataColumn("TaskInstanceId", typeof(string)));
dt.Columns.Add(new DataColumn("ExecutionUid", typeof(Guid)));
dt.Columns.Add(new DataColumn("PipelineName", typeof(string)));
dt.Columns.Add(new DataColumn("DatafactorySubscriptionUid", typeof(Guid)));
dt.Columns.Add(new DataColumn("DatafactoryResourceGroup", typeof(string)));
dt.Columns.Add(new DataColumn("DatafactoryName", typeof(string)));
dt.Columns.Add(new DataColumn("RunUid", typeof(Guid)));
dt.Columns.Add(new DataColumn("Status", typeof(string)));
dt.Columns.Add(new DataColumn("SimpleStatus", typeof(string)));
//Check Each Running Pipeline
foreach (dynamic _Pipeline in _ActivePipelines)
{
dynamic _PipelineStatus = CheckPipelineStatus.CheckPipelineStatusMethod(_Pipeline.DatafactorySubscriptionUid.ToString(), _Pipeline.DatafactoryResourceGroup.ToString(), _Pipeline.DatafactoryName.ToString(), _Pipeline.PipelineName.ToString(), _Pipeline.AdfRunUid.ToString(), logging);
if (_PipelineStatus["SimpleStatus"].ToString() == "Runnning")
{
RunningPipelines += 1;
}
if (_PipelineStatus["SimpleStatus"].ToString() == "Done")
{
FinishedPipelines += 1;
}
DataRow dr = dt.NewRow();
dr["TaskInstanceId"] = _Pipeline.TaskInstanceId;
dr["ExecutionUid"] = _Pipeline.ExecutionUid;
dr["DatafactorySubscriptionUid"] = _Pipeline.DatafactorySubscriptionUid;
dr["DatafactoryResourceGroup"] = _Pipeline.DatafactoryResourceGroup;
dr["DatafactoryName"] = _Pipeline.DatafactoryName;
dr["Status"] = _PipelineStatus["Status"];
dr["SimpleStatus"] = _PipelineStatus["SimpleStatus"];
dr["RunUid"] = (Guid)_PipelineStatus["RunId"];
dr["PipelineName"] = _PipelineStatus["PipelineName"];
dt.Rows.Add(dr);
}
string TempTableName = "#Temp" + Guid.NewGuid().ToString();
TaskMetaDataDatabase TMD = new TaskMetaDataDatabase();
//Todo: Update both the TaskInstanceExecution and the TaskInstance;
TMD.AutoBulkInsertAndMerge(dt, TempTableName, "TaskInstanceExecution");
return(RunningPipelines);
}
public static JObject UpdateWaterMark(HttpRequest req,
Logging LogHelper)
{
string requestBody = new StreamReader(req.Body).ReadToEndAsync().Result;
dynamic data = JsonConvert.DeserializeObject(requestBody);
dynamic _TaskMasterId = JObject.Parse(data.ToString())["TaskMasterId"];
dynamic _TaskMasterWaterMarkColumnType = JObject.Parse(data.ToString())["TaskMasterWaterMarkColumnType"];
dynamic _WaterMarkValue = JObject.Parse(data.ToString())["WaterMarkValue"];
if (!string.IsNullOrEmpty(_WaterMarkValue.ToString()))
{
DataTable dt = new DataTable();
dt.Columns.Add(new DataColumn("TaskMasterId", typeof(long)));
dt.Columns.Add(new DataColumn("TaskMasterWaterMarkColumnType", typeof(string)));
dt.Columns.Add(new DataColumn("TaskMasterWaterMark_DateTime", typeof(DateTime)));
dt.Columns.Add(new DataColumn("TaskMasterWaterMark_BigInt", typeof(long)));
dt.Columns.Add(new DataColumn("ActiveYN", typeof(bool)));
dt.Columns.Add(new DataColumn("UpdatedOn", typeof(DateTime)));
DataRow dr = dt.NewRow();
dr["TaskMasterId"] = _TaskMasterId;
dr["TaskMasterWaterMarkColumnType"] = _TaskMasterWaterMarkColumnType;
if (_TaskMasterWaterMarkColumnType == "DateTime")
{
dr["TaskMasterWaterMark_DateTime"] = _WaterMarkValue;
dr["TaskMasterWaterMark_BigInt"] = DBNull.Value;
}
else if (_TaskMasterWaterMarkColumnType == "BigInt")
{
dr["TaskMasterWaterMark_DateTime"] = DBNull.Value;
dr["TaskMasterWaterMark_BigInt"] = _WaterMarkValue;
}
else
{
throw new System.ArgumentException(string.Format("Invalid WaterMark ColumnType = '{0}'", _TaskMasterWaterMarkColumnType));
}
dr["ActiveYN"] = 1;
dr["UpdatedOn"] = DateTime.UtcNow;
dt.Rows.Add(dr);
string TempTableName = "#Temp_TaskMasterWaterMark_" + Guid.NewGuid().ToString();
TaskMetaDataDatabase TMD = new TaskMetaDataDatabase();
TMD.AutoBulkInsertAndMerge(dt, TempTableName, "TaskMasterWaterMark");
}
JObject Root = new JObject
{
["Result"] = "Complete"
};
return(Root);
}
public static void QueryPipelineRuns(string subscriptionId, string resourceGroup, string factoryName, string _rungroupid, DateTime startDT, DateTime endDT, Logging logging)
{
#region QueryPipelineRuns
logging.LogInformation("Query ADF Pipeline Runs.");
string outputString = string.Empty;
DataTable dt = new DataTable();
dt.Columns.Add(new DataColumn("ExecutionUid", typeof(Guid)));
dt.Columns.Add(new DataColumn("TaskInstanceId", typeof(Int64)));
dt.Columns.Add(new DataColumn("TaskMasterId", typeof(Int64)));
//dt.Columns.Add(new DataColumn("AdditionalProperties", typeof(String)));
dt.Columns.Add(new DataColumn("DurationInMs", typeof(Int64)));
//dt.Columns.Add(new DataColumn("InvokedBy", typeof(String)));
dt.Columns.Add(new DataColumn("IsLastest", typeof(Boolean)));
dt.Columns.Add(new DataColumn("LastUpdated", typeof(DateTime)));
//dt.Columns.Add(new DataColumn("Message", typeof(String)));
//dt.Columns.Add(new DataColumn("Parameters", typeof(String)));
dt.Columns.Add(new DataColumn("RunId", typeof(Guid)));
//dt.Columns.Add(new DataColumn("RunGroupId", typeof(Guid)));
dt.Columns.Add(new DataColumn("PipelineName", typeof(String)));
dt.Columns.Add(new DataColumn("RunStart", typeof(DateTime)));
dt.Columns.Add(new DataColumn("RunEnd", typeof(DateTime)));
dt.Columns.Add(new DataColumn("RunDimensions", typeof(String)));
dt.Columns.Add(new DataColumn("Status", typeof(String)));
DataTable ActivityDt = new DataTable();
ActivityDt.Columns.Add(new DataColumn("ActivityName", typeof(String)));
ActivityDt.Columns.Add(new DataColumn("RunId", typeof(Guid)));
ActivityDt.Columns.Add(new DataColumn("ActivityRunStart", typeof(DateTime)));
ActivityDt.Columns.Add(new DataColumn("ActivityRunEnd", typeof(DateTime)));
ActivityDt.Columns.Add(new DataColumn("ActivityRunId", typeof(Guid)));
ActivityDt.Columns.Add(new DataColumn("ActivityType", typeof(String)));
//dt.Columns.Add(new DataColumn("AdditionalProperties", typeof(String)));
ActivityDt.Columns.Add(new DataColumn("DurationInMs", typeof(Int64)));
//dt.Columns.Add(new DataColumn("Error", typeof(String)));
//dt.Columns.Add(new DataColumn("Input", typeof(String)));
//dt.Columns.Add(new DataColumn("LinkedServiceName", typeof(String)));
ActivityDt.Columns.Add(new DataColumn("OutPut", typeof(String)));
ActivityDt.Columns.Add(new DataColumn("PipelineName", typeof(String)));
ActivityDt.Columns.Add(new DataColumn("PipelineRunId", typeof(String)));
ActivityDt.Columns.Add(new DataColumn("Status", typeof(String)));
using (var client = DataFactoryClient.CreateDataFactoryClient(subscriptionId))
{
//Get pipeline status with provided run id
PipelineRunsQueryResponse pipelineRunsQueryResponse;
RunFilterParameters filterParameterActivityRuns = new RunFilterParameters();
filterParameterActivityRuns.LastUpdatedAfter = startDT;
filterParameterActivityRuns.LastUpdatedBefore = endDT.AddHours(+2);
RunFilterParameters filterParameter = new RunFilterParameters();
filterParameter.LastUpdatedAfter = startDT;
filterParameter.LastUpdatedBefore = endDT;
IList <string> rungroupid = new List <string> {
_rungroupid
};
IList <RunQueryFilter> filter = new List <RunQueryFilter>();
filter.Add(new RunQueryFilter
{
Operand = RunQueryFilterOperand.RunGroupId,
OperatorProperty = RunQueryFilterOperator.Equals,
Values = rungroupid
});
filterParameter.Filters = filter;
logging.LogInformation(String.Format("API PipelineRuns.QueryByFactory Start"));
pipelineRunsQueryResponse = client.PipelineRuns.QueryByFactory(resourceGroup, factoryName, filterParameter);
logging.LogInformation(String.Format("API PipelineRuns.QueryByFactory End"));
var enumerator = pipelineRunsQueryResponse.Value.GetEnumerator();
PipelineRun pipelineRuns;
string runId = String.Empty;
int item = 0;
while (true)
{
for (bool hasMoreRuns = enumerator.MoveNext(); hasMoreRuns;)
{
pipelineRuns = enumerator.Current;
hasMoreRuns = enumerator.MoveNext();
runId = pipelineRuns.RunId;
item += 1;
logging.LogInformation(String.Format("PipelineRuns.QueryByFactory RunId {0} Current Item {1} of {2}", runId, item, pipelineRunsQueryResponse.Value.Count));
DataRow dr = dt.NewRow();
string _param = string.Empty;
foreach (var element in pipelineRuns.Parameters)
{
_param = element.Value;
break;
}
dr["ExecutionUid"] = Shared.JsonHelpers.GetStringValueFromJSON(logging, "ExecutionUid", JObject.Parse(_param), null, true);
dr["TaskInstanceId"] = Shared.JsonHelpers.GetStringValueFromJSON(logging, "TaskInstanceId", JObject.Parse(_param), null, true);
dr["TaskMasterId"] = Shared.JsonHelpers.GetStringValueFromJSON(logging, "TaskMasterId", JObject.Parse(_param), null, true);
//dr["AdditionalProperties"] = pipelineRuns.AdditionalProperties ?? (object)DBNull.Value;
dr["DurationInMs"] = pipelineRuns.DurationInMs ?? (object)DBNull.Value;
//dr["InvokedBy"] = pipelineRuns.InvokedBy ?? (object)DBNull.Value;
dr["IsLastest"] = pipelineRuns.IsLatest ?? (object)DBNull.Value;
dr["LastUpdated"] = pipelineRuns.LastUpdated ?? (object)DBNull.Value;
//dr["Message"] = pipelineRuns.Message ?? (object)DBNull.Value;
//dr["Parameters"] = _param;
dr["RunId"] = pipelineRuns.RunId;
//dr["RunGroupId"] = pipelineRuns.RunGroupId ?? (object)DBNull.Value;
dr["PipelineName"] = pipelineRuns.PipelineName ?? (object)DBNull.Value;
dr["RunStart"] = pipelineRuns.RunStart ?? (object)DBNull.Value;
dr["RunEnd"] = pipelineRuns.RunEnd ?? (object)DBNull.Value;
dr["RunDimensions"] = pipelineRuns.PipelineName ?? (object)DBNull.Value;
dr["Status"] = pipelineRuns.Status ?? (object)DBNull.Value;
dt.Rows.Add(dr);
QueryActivityRuns(subscriptionId, resourceGroup, factoryName, runId, runId, filterParameterActivityRuns, logging, ref ActivityDt);
}
if (pipelineRunsQueryResponse.ContinuationToken == null)
{
break;
}
filterParameter.ContinuationToken = pipelineRunsQueryResponse.ContinuationToken;
pipelineRunsQueryResponse = client.PipelineRuns.QueryByFactory(resourceGroup, factoryName, filterParameter);
enumerator = pipelineRunsQueryResponse.Value.GetEnumerator();
item = 0;
}
}
if (ActivityDt.Rows.Count > 0)
{
string TempTableName = "#Temp_ADFActivities_" + Guid.NewGuid().ToString();
TaskMetaDataDatabase TMD = new TaskMetaDataDatabase();
TMD.AutoBulkInsertAndMerge(ActivityDt, TempTableName, "ADFActivity");
}
if (dt.Rows.Count > 0)
{
string TempTableName = "#Temp_ADFPipelineRun_" + Guid.NewGuid().ToString();
TaskMetaDataDatabase TMD = new TaskMetaDataDatabase();
TMD.AutoBulkInsertAndMerge(dt, TempTableName, "ADFPipelineRun");
}
#endregion
}
