public static dynamic PrepareFrameworkTasksCore(Logging logging)
{
TaskMetaDataDatabase TMD = new TaskMetaDataDatabase();
TMD.ExecuteSql(string.Format("Insert into Execution values ('{0}', '{1}', '{2}')", logging.DefaultActivityLogItem.ExecutionUid, DateTimeOffset.Now.ToString("u"), DateTimeOffset.Now.AddYears(999).ToString("u")));
//Check status of running pipelines and calculate available "slots" based on max concurrency settings
short _FrameworkWideMaxConcurrency = Shared.GlobalConfigs.GetInt16Config("FrameworkWideMaxConcurrency");
//ToDo: Write Pipelines that need to be checked to Queue for now I have just reduced to only those tasks that have been running for longer than x minutes.
//CheckLongRunningPipelines(logging);
//Get Count of All runnning pipelines directly from the database
short _RunnningPipelines = CountRunnningPipelines(logging);
short _AvailableConcurrencySlots = (short)(_FrameworkWideMaxConcurrency - _RunnningPipelines);
//Generate new task instances based on task master and schedules
CreateTaskInstance(logging);
//Is there is Available Slots Proceed
if (_AvailableConcurrencySlots > 0)
{
List <AdsGoFast.TaskMetaData.TaskGroup> _TaskGroups = TaskGroupsStatic.GetActive();
if (_TaskGroups.Count > 0)
{
short _ConcurrencySlotsAllocated = 0;
short _DefaultTasksPerGroup = 0;
short _DistributionLoopCounter = 1;
//Distribute Concurrency Slots
while (_AvailableConcurrencySlots > 0)
{
DistributeConcurrencySlots(ref _TaskGroups, ref _DefaultTasksPerGroup, ref _ConcurrencySlotsAllocated, ref _AvailableConcurrencySlots, _DistributionLoopCounter);
_DistributionLoopCounter += 1;
}
Table TempTarget = new Table
{
Schema = "dbo",
Name = "#TempGroups" + logging.DefaultActivityLogItem.ExecutionUid.ToString()
};
SqlConnection _con = TMD.GetSqlConnection();
TMD.BulkInsert(_TaskGroups.ToDataTable(), TempTarget, true, _con);
Dictionary <string, string> _params = new Dictionary <string, string>
{
{ "TempTable", TempTarget.QuotedSchemaAndName() }
};
string _sql = GenerateSQLStatementTemplates.GetSQL(Shared.GlobalConfigs.GetStringConfig("SQLTemplateLocation"), "UpdateTaskInstancesWithTaskRunner", _params);
TMD.ExecuteSql(_sql, _con);
}
}
return(new { });
}
private void SaveCursor(string cursor)
{
TaskMetaDataDatabase TMD = new TaskMetaDataDatabase();
using SqlConnection _con = TMD.GetSqlConnection();
TMD.ExecuteSql("Select null");
}
public async Task <IActionResult> Run([HttpTrigger(AuthorizationLevel.Anonymous, "get", Route = null)] HttpRequest req, ILogger log, ExecutionContext context, System.Security.Claims.ClaimsPrincipal principal)
{
bool IsAuthorised = _sap.IsAuthorised(req, log);
if (IsAuthorised)
{
Guid ExecutionId = context.InvocationId;
using FrameworkRunner FR = new FrameworkRunner(log, ExecutionId);
FrameworkRunner.FrameworkRunnerWorkerWithHttpRequest worker = RunFrameworkTasks.RunFrameworkTasksCore;
FrameworkRunner.FrameworkRunnerResult result = FR.Invoke(req, "RunFrameworkTasksHttpTrigger", worker);
if (result.Succeeded)
{
return(new OkObjectResult(JObject.Parse(result.ReturnObject)));
}
else
{
return(new BadRequestObjectResult(new { Error = "Execution Failed...." }));
}
}
else
{
log.LogWarning("User is not authorised to call RunFrameworkTasksHttpTrigger.");
TaskMetaDataDatabase TMD = new TaskMetaDataDatabase();
short TaskRunnerId = System.Convert.ToInt16(req.Query["TaskRunnerId"]);
TMD.ExecuteSql(string.Format("exec [dbo].[UpdFrameworkTaskRunner] {0}", TaskRunnerId));
return(new BadRequestObjectResult(new { Error = "User is not authorised to call this API...." }));
}
}
public static dynamic Testing_MarkTasksCompleteCore(Logging logging)
{
TaskMetaDataDatabase TMD = new TaskMetaDataDatabase();
TMD.ExecuteSql(string.Format("Insert into Execution values ('{0}', '{1}', '{2}')", logging.DefaultActivityLogItem.ExecutionUid, DateTimeOffset.Now.ToString("u"), DateTimeOffset.Now.AddYears(999).ToString("u")));
int res = TMD.GetSqlConnection().Execute(@"
Declare @RunningTasks int = (
Select Count(*)
from TaskInstance
where TaskRunnerId is not null aND LastExecutionStatus = 'InProgress')
Declare @TasksToBeMarkedComplete int = (
SELECT ROUND(1 + (RAND() * @RunningTasks),0) AS RAND_1_100)
Update TaskInstance
Set LastExecutionStatus = 'Completed', TaskRunnerId = null
from TaskInstance a
inner join
(
Select *, Rn = ROW_NUMBER() over (order by TaskInstanceId)
from TaskInstance
where TaskRunnerId is not null aND LastExecutionStatus = 'InProgress'
) b on a.TaskInstanceId = b.TaskInstanceId
where a.TaskRunnerId is not null and a.LastExecutionStatus = 'InProgress'
and b.Rn <= @TasksToBeMarkedComplete
");
return(new { });
}
public static JObject GetSQLMergeStatementCore(HttpRequest req,
Logging logging)
{
string requestBody = new StreamReader(req.Body).ReadToEndAsync().Result;
dynamic data = JsonConvert.DeserializeObject(requestBody);
JObject Root = new JObject();
TaskMetaDataDatabase TMD = new TaskMetaDataDatabase();
using (SqlConnection _con = TMD.GetSqlConnection())
{
string _token = Shared._AzureAuthenticationCredentialProvider.GetAzureRestApiToken("https://database.windows.net/");
String g = Guid.NewGuid().ToString().Replace("-", "");
_con.AccessToken = _token;
JArray arrStage = (JArray)data["Stage"];
string _StagingTableSchema = data["StagingTableSchema"].ToString();
string _StagingTableName = "#Temp_" + data["StagingTableName"].ToString() + g.ToString();
string _CreateStatementStage = GenerateSQLStatementTemplates.GetCreateTable(arrStage, _StagingTableSchema, _StagingTableName, false);
TMD.ExecuteSql(_CreateStatementStage, _con);
JArray arrTarget = (JArray)data["Target"];
string _TargetTableSchema = data["TargetTableSchema"].ToString();
string _TargetTableName = "#Temp_" + data["TargetTableName"].ToString() + g.ToString();
string _CreateStatementTarget = GenerateSQLStatementTemplates.GetCreateTable(arrTarget, _TargetTableSchema, _TargetTableName, false);
TMD.ExecuteSql(_CreateStatementTarget, _con);
string _MergeStatement = TMD.GenerateMergeSQL(_StagingTableSchema, _StagingTableName, _TargetTableSchema, _TargetTableName, _con, true, logging);
string fullStagingTableName = string.Format("[{0}].[{1}]", _StagingTableSchema, _StagingTableName.Replace("#Temp_", "").Replace(g.ToString(), ""));
string fullTargetTableName = string.Format("[{0}].[{1}]", _TargetTableSchema, _TargetTableName.Replace("#Temp_", "").Replace(g.ToString(), ""));
_MergeStatement = _MergeStatement.Replace(_TargetTableName, fullTargetTableName);
_MergeStatement = _MergeStatement.Replace(_StagingTableName, fullStagingTableName);
//Add Select for ADF Lookup Activity
_MergeStatement += Environment.NewLine + "Select 1 ";
Root["MergeStatement"] = _MergeStatement;
logging.LogInformation("GetSQLMergeStatement Function complete.");
}
return(Root);
}
public static dynamic RunFrameworkTasksCore(HttpRequest req, Logging logging)
{
TaskMetaDataDatabase TMD = new TaskMetaDataDatabase();
short TaskRunnerId = System.Convert.ToInt16(req.Query["TaskRunnerId"]);
try
{
TMD.ExecuteSql(string.Format("Insert into Execution values ('{0}', '{1}', '{2}')", logging.DefaultActivityLogItem.ExecutionUid, DateTimeOffset.Now.ToString("u"), DateTimeOffset.Now.AddYears(999).ToString("u")));
//Fetch Top # tasks
JArray _Tasks = AdsGoFast.TaskMetaData.TaskInstancesStatic.GetActive_ADFJSON((Guid)logging.DefaultActivityLogItem.ExecutionUid, TaskRunnerId, logging);
var UtcCurDay = DateTime.UtcNow.ToString("yyyyMMdd");
foreach (JObject _Task in _Tasks)
{
long _TaskInstanceId = System.Convert.ToInt64(Shared.JsonHelpers.GetDynamicValueFromJSON(logging, "TaskInstanceId", _Task, null, true));
logging.DefaultActivityLogItem.TaskInstanceId = _TaskInstanceId;
//TO DO: Update TaskInstance yto UnTried if failed
string _pipelinename = _Task["DataFactory"]["ADFPipeline"].ToString();
System.Collections.Generic.Dictionary <string, object> _pipelineparams = new System.Collections.Generic.Dictionary <string, object>();
logging.LogInformation(string.Format("Executing ADF Pipeline for TaskInstanceId {0} ", _TaskInstanceId.ToString()));
//Check Task Type and execute appropriate ADF Pipeline
//Todo: Potentially extract switch into metadata
if (Shared._ApplicationOptions.TestingOptions.GenerateTaskObjectTestFiles)
{
string FileFullPath = Shared._ApplicationOptions.TestingOptions.TaskObjectTestFileLocation + /*UtcCurDay +*/ "/";
// Determine whether the directory exists.
if (!System.IO.Directory.Exists(FileFullPath))
{
// Try to create the directory.
System.IO.DirectoryInfo di = System.IO.Directory.CreateDirectory(FileFullPath);
}
FileFullPath = FileFullPath + _Task["TaskType"].ToString() + "_" + _pipelinename.ToString() + "_" + _Task["TaskMasterId"].ToString() + ".json";
System.IO.File.WriteAllText(FileFullPath, _Task.ToString());
TMD.LogTaskInstanceCompletion(_TaskInstanceId, (Guid)logging.DefaultActivityLogItem.ExecutionUid, TaskMetaData.BaseTasks.TaskStatus.Complete, System.Guid.Empty, "Complete");
}
else
{
try
{
if (_Task["TaskExecutionType"].ToString() == "ADF")
{
_pipelinename = "Master";
_pipelineparams.Add("TaskObject", _Task);
if (_pipelinename != "")
{
JObject _pipelineresult = ExecutePipeline.ExecutePipelineMethod(_Task["DataFactory"]["SubscriptionId"].ToString(), _Task["DataFactory"]["ResourceGroup"].ToString(), _Task["DataFactory"]["Name"].ToString(), _pipelinename, _pipelineparams, logging);
logging.DefaultActivityLogItem.AdfRunUid = Guid.Parse(_pipelineresult["RunId"].ToString());
TMD.GetSqlConnection().Execute(string.Format(@"
INSERT INTO TaskInstanceExecution (
[ExecutionUid]
,[TaskInstanceId]
,[DatafactorySubscriptionUid]
,[DatafactoryResourceGroup]
,[DatafactoryName]
,[PipelineName]
,[AdfRunUid]
,[StartDateTime]
,[Status]
,[Comment]
)
VALUES (
@ExecutionUid
,@TaskInstanceId
,@DatafactorySubscriptionUid
,@DatafactoryResourceGroup
,@DatafactoryName
,@PipelineName
,@AdfRunUid
,@StartDateTime
,@Status
,@Comment
)" ), new
{
ExecutionUid = logging.DefaultActivityLogItem.ExecutionUid.ToString(),
TaskInstanceId = System.Convert.ToInt64(_Task["TaskInstanceId"]),
DatafactorySubscriptionUid = _Task["DataFactory"]["SubscriptionId"].ToString(),
DatafactoryResourceGroup = _Task["DataFactory"]["ResourceGroup"].ToString(),
DatafactoryName = _Task["DataFactory"]["Name"].ToString(),
PipelineName = _pipelineresult["PipelineName"].ToString(),
AdfRunUid = Guid.Parse(_pipelineresult["RunId"].ToString()),
StartDateTime = DateTimeOffset.UtcNow,
Status = _pipelineresult["Status"].ToString(),
Comment = ""
});
}
//To Do // Batch to make less "chatty"
//To Do // Upgrade to stored procedure call
}
else if (_Task["TaskExecutionType"].ToString() == "AF")
{
//The "AF" branch is for calling Azure Function Based Tasks that do not require ADF. Calls are made async (just like the ADF calls) and calls are made using "AsyncHttp" requests even though at present the "AF" based Tasks reside in the same function app. This is to "future proof" as it is expected that these AF based tasks will be moved out to a separate function app in the future.
switch (_pipelinename)
{
case "AZ-Storage-SAS-Uri-SMTP-Email":
using (var client = new System.Net.Http.HttpClient())
{
//Lets get an access token based on MSI or Service Principal
var secureFunctionAPIURL = string.Format("{0}/api/GetSASUriSendEmailHttpTrigger", Shared._ApplicationOptions.ServiceConnections.CoreFunctionsURL);
var accessToken = Shared._AzureAuthenticationCredentialProvider.GetAzureRestApiToken(secureFunctionAPIURL);
using HttpRequestMessage httpRequestMessage = new HttpRequestMessage
{
Method = HttpMethod.Post,
RequestUri = new Uri(secureFunctionAPIURL),
Content = new StringContent(_Task.ToString(), System.Text.Encoding.UTF8, "application/json"),
Headers = { { System.Net.HttpRequestHeader.Authorization.ToString(), "Bearer " + accessToken } }
};
//Todo Add some error handling in case function cannot be reached. Note Wait time is there to provide sufficient time to complete post before the HttpClient is disposed.
var HttpTask = client.SendAsync(httpRequestMessage).Wait(3000);
}
break;
case "AZ-Storage-Cache-File-List":
using (var client = new System.Net.Http.HttpClient())
{
//Lets get an access token based on MSI or Service Principal
var secureFunctionAPIURL = string.Format("{0}/api/AZStorageCacheFileListHttpTrigger", Shared._ApplicationOptions.ServiceConnections.CoreFunctionsURL);
var accessToken = Shared._AzureAuthenticationCredentialProvider.GetAzureRestApiToken(secureFunctionAPIURL);
using HttpRequestMessage httpRequestMessage = new HttpRequestMessage
{
Method = HttpMethod.Post,
RequestUri = new Uri(secureFunctionAPIURL),
Content = new StringContent(_Task.ToString(), System.Text.Encoding.UTF8, "application/json"),
Headers = { { System.Net.HttpRequestHeader.Authorization.ToString(), "Bearer " + accessToken } }
};
//Todo Add some error handling in case function cannot be reached. Note Wait time is there to provide sufficient time to complete post before the HttpClient is disposed.
var HttpTask = client.SendAsync(httpRequestMessage).Wait(3000);
}
break;
case "StartAndStopVMs":
using (var client = new System.Net.Http.HttpClient())
{
//Lets get an access token based on MSI or Service Principal
var accessToken = GetSecureFunctionToken(_pipelinename);
using HttpRequestMessage httpRequestMessage = new HttpRequestMessage
{
Method = HttpMethod.Post,
RequestUri = new Uri(GetSecureFunctionURI(_pipelinename)),
Content = new StringContent(_Task.ToString(), System.Text.Encoding.UTF8, "application/json"),
Headers = { { System.Net.HttpRequestHeader.Authorization.ToString(), "Bearer " + accessToken } }
};
//Todo Add some error handling in case function cannot be reached. Note Wait time is there to provide sufficient time to complete post before the HttpClient is disposed.
var HttpTask = client.SendAsync(httpRequestMessage).Wait(3000);
}
break;
case "Cache-File-List-To-Email-Alert":
using (var client = new System.Net.Http.HttpClient())
{
SendAlert(_Task, logging);
}
break;
default:
var msg = $"Could not find execution path for Task Type of {_pipelinename} and Execution Type of {_Task["TaskExecutionType"].ToString()}";
logging.LogErrors(new Exception(msg));
TMD.LogTaskInstanceCompletion((Int64)_TaskInstanceId, (System.Guid)logging.DefaultActivityLogItem.ExecutionUid, BaseTasks.TaskStatus.FailedNoRetry, Guid.Empty, (String)msg);
break;
}
//To Do // Batch to make less "chatty"
//To Do // Upgrade to stored procedure call
}
}
catch (Exception TaskException)
{
logging.LogErrors(TaskException);
TMD.LogTaskInstanceCompletion((Int64)_TaskInstanceId, (System.Guid)logging.DefaultActivityLogItem.ExecutionUid, BaseTasks.TaskStatus.FailedNoRetry, Guid.Empty, (String)"Runner failed to execute task.");
}
}
}
}
catch (Exception RunnerException)
{
//Set Runner back to Idle
TMD.ExecuteSql(string.Format("exec [dbo].[UpdFrameworkTaskRunner] {0}", TaskRunnerId));
logging.LogErrors(RunnerException);
//log and re-throw the error
throw RunnerException;
}
//Set Runner back to Idle
TMD.ExecuteSql(string.Format("exec [dbo].[UpdFrameworkTaskRunner] {0}", TaskRunnerId));
//Return success
JObject Root = new JObject
{
["Succeeded"] = true
};
return(Root);
}