public static async Task ProcessNotificationAsync(WorkerRequest request, ProcessNotificationRequest @event)
{
var jobProcessId = @event.JobProcessId;
var notification = @event.Notification;
var notificationJobData = notification.Content.ToMcmaObject <JobBase>();
var table = new DynamoDbTable <JobProcess>(request.TableName());
var jobProcess = await table.GetAsync(jobProcessId);
// not updating job if it already was marked as completed or failed.
if (jobProcess.Status == JobStatus.Completed || jobProcess.Status == JobStatus.Failed)
{
Logger.Warn("Ignoring update of job process that tried to change state from " + jobProcess.Status + " to " + notificationJobData.Status);
return;
}
jobProcess.Status = notificationJobData.Status;
jobProcess.StatusMessage = notificationJobData.StatusMessage;
jobProcess.Progress = notificationJobData.Progress;
jobProcess.JobOutput = notificationJobData.JobOutput;
jobProcess.DateModified = DateTime.UtcNow;
await table.PutAsync(jobProcessId, jobProcess);
var resourceManager = request.GetAwsV4ResourceManager();
await resourceManager.SendNotificationAsync(jobProcess, jobProcess.NotificationEndpoint);
}
internal static async Task ProcessNotificationAsync(WorkflowServiceWorkerRequest @event)
{
var jobAssignmentId = @event.JobAssignmentId;
var notification = @event.Notification;
var notificationJobPayload = notification.Content.ToMcmaObject <JobBase>();
var table = new DynamoDbTable(@event.Request.StageVariables["TableName"]);
var jobAssignment = await table.GetAsync <JobAssignment>(jobAssignmentId);
jobAssignment.Status = notificationJobPayload.Status;
jobAssignment.StatusMessage = notificationJobPayload.StatusMessage;
if (notificationJobPayload.Progress != null)
{
jobAssignment.Progress = notificationJobPayload.Progress;
}
jobAssignment.JobOutput = notificationJobPayload.JobOutput;
jobAssignment.DateModified = DateTime.UtcNow;
await table.PutAsync <JobAssignment>(jobAssignmentId, jobAssignment);
var resourceManager = @event.Request.GetAwsV4ResourceManager();
await resourceManager.SendNotificationAsync(jobAssignment, jobAssignment.NotificationEndpoint);
}
public static async Task ProcessNotificationAsync(McmaApiRequestContext requestContext)
{
var table = new DynamoDbTable <JobAssignment>(requestContext.TableName());
var jobAssignment =
await table.GetAsync(requestContext.PublicUrl() + "/job-assignments/" + requestContext.Request.PathVariables["id"]);
if (!requestContext.ResourceIfFound(jobAssignment, false) || requestContext.IsBadRequestDueToMissingBody(out Notification notification))
{
return;
}
await WorkerInvoker.RunAsync(
requestContext.WorkerFunctionName(),
new
{
operationName = "ProcessNotification",
contextVariables = requestContext.GetAllContextVariables(),
input = new
{
jobAssignmentId = jobAssignment.Id,
notification = notification
}
});
}
public static async Task DeleteJobAsync(ApiGatewayRequest request, McmaApiResponse response)
{
Logger.Debug(nameof(DeleteJobAsync));
Logger.Debug(request.ToMcmaJson().ToString());
var table = new DynamoDbTable(request.StageVariables["TableName"]);
var jobId = request.StageVariables["PublicUrl"] + request.Path;
var job = await table.GetAsync <Job>(jobId);
if (job == null)
{
response.StatusCode = (int)HttpStatusCode.NotFound;
response.StatusMessage = "No resource found on path '" + request.Path + "'.";
return;
}
await table.DeleteAsync <Job>(jobId);
// invoking worker lambda function that will delete the JobProcess created for this Job
if (!string.IsNullOrEmpty(job.JobProcess))
{
var lambdaClient = new AmazonLambdaClient();
var invokeRequest = new InvokeRequest
{
FunctionName = request.StageVariables["WorkerLambdaFunctionName"],
InvocationType = "Event",
LogType = "None",
Payload = new { action = "deleteJobProcess", request = request, jobProcessId = job.JobProcess }.ToMcmaJson().ToString()
};
await lambdaClient.InvokeAsync(invokeRequest);
}
}
public static async Task ProcessNotificationAsync(McmaApiRequestContext requestContext)
{
var request = requestContext.Request;
var response = requestContext.Response;
var table = new DynamoDbTable <JobProcess>(requestContext.TableName());
var jobProcess = await table.GetAsync(requestContext.PublicUrl() + "/job-processes/" + request.PathVariables["id"]);
if (!requestContext.ResourceIfFound(jobProcess, false) ||
requestContext.IsBadRequestDueToMissingBody <Notification>(out var notification))
{
return;
}
if (jobProcess.JobAssignment != notification.Source)
{
response.StatusCode = (int)HttpStatusCode.BadRequest;
response.StatusMessage = "Unexpected notification from '" + notification.Source + "'.";
return;
}
await WorkerInvoker.RunAsync(
requestContext.WorkerFunctionName(),
new
{
operationName = "processNotification",
contextVariables = requestContext.GetAllContextVariables(),
input = new
{
jobProcessId = jobProcess.Id,
notification = notification
}
});
}
internal static async Task ProcessNotificationAsync(JobRepositoryWorkerRequest @event)
{
var jobId = @event.JobId;
var notification = @event.Notification;
var notificationJob = notification.Content.ToMcmaObject <JobBase>();
var table = new DynamoDbTable(@event.Request.StageVariables["TableName"]);
var job = await table.GetAsync <Job>(jobId);
// not updating job if it already was marked as completed or failed.
if (job.Status == "COMPLETED" || job.Status == "FAILED")
{
Logger.Warn("Ignoring update of job that tried to change state from " + job.Status + " to " + notificationJob.Status);
return;
}
job.Status = notificationJob.Status;
job.StatusMessage = notificationJob.StatusMessage;
job.Progress = notificationJob.Progress;
job.JobOutput = notificationJob.JobOutput;
job.DateModified = DateTime.UtcNow;
await table.PutAsync <Job>(jobId, job);
var resourceManager = @event.Request.GetAwsV4ResourceManager();
await resourceManager.SendNotificationAsync(job, job.NotificationEndpoint);
}
protected override async Task ExecuteAsync(WorkerRequest @event, ProcessNotificationRequest notificationRequest)
{
var jobAssignmentId = notificationRequest.JobAssignmentId;
var notification = notificationRequest.Notification;
var notificationJobPayload = notification.Content.ToMcmaObject <JobBase>();
var table = new DynamoDbTable <JobAssignment>(@event.TableName());
var jobAssignment = await table.GetAsync(jobAssignmentId);
jobAssignment.Status = notificationJobPayload.Status;
jobAssignment.StatusMessage = notificationJobPayload.StatusMessage;
if (notificationJobPayload.Progress != null)
{
jobAssignment.Progress = notificationJobPayload.Progress;
}
jobAssignment.JobOutput = notificationJobPayload.JobOutput;
jobAssignment.DateModified = DateTime.UtcNow;
await table.PutAsync(jobAssignmentId, jobAssignment);
var resourceManager = @event.GetAwsV4ResourceManager();
await resourceManager.SendNotificationAsync(jobAssignment, jobAssignment.NotificationEndpoint);
}
public static async Task ProcessNotificationAsync(ApiGatewayRequest request, McmaApiResponse response)
{
var table = new DynamoDbTable(request.StageVariables["TableName"]);
var jobAssignmentId = request.StageVariables["PublicUrl"] + "/job-assignments/" + request.PathVariables["id"];
var jobAssignment = await table.GetAsync <JobAssignment>(jobAssignmentId);
if (jobAssignment == null)
{
response.StatusCode = (int)HttpStatusCode.NotFound;
response.StatusMessage = "No resource found on path '" + request.Path + "'";
return;
}
var notification = request.JsonBody?.ToMcmaObject <Notification>();
if (notification == null)
{
response.StatusCode = (int)HttpStatusCode.BadRequest;
response.StatusMessage = "Missing notification in request body";
return;
}
var lambdaClient = new AmazonLambdaClient();
var invokeRequest = new InvokeRequest
{
FunctionName = request.StageVariables["WorkerLambdaFunctionName"],
InvocationType = "Event",
LogType = "None",
Payload = new { action = "ProcessNotification", request = request, jobAssignmentId = jobAssignmentId, notification = notification }.ToMcmaJson().ToString()
};
await lambdaClient.InvokeAsync(invokeRequest);
}
private static async Task <JobAssignment> GetJobAssignmentAsync(DynamoDbTable table, string jobAssignmentId)
{
var jobAssignment = await table.GetAsync <JobAssignment>(jobAssignmentId);
if (jobAssignment == null)
{
throw new Exception("JobAssignment with id '" + jobAssignmentId + "' not found");
}
return(jobAssignment);
}
public static async Task ProcessNotificationAsync(ApiGatewayRequest request, McmaApiResponse response)
{
Logger.Debug(nameof(ProcessNotificationAsync));
Logger.Debug(request.ToMcmaJson().ToString());
var table = new DynamoDbTable(request.StageVariables["TableName"]);
var jobAssignmentId = request.StageVariables["PublicUrl"] + "/job-assignments/" + request.PathVariables["id"];
var jobAssignment = await table.GetAsync <JobAssignment>(jobAssignmentId);
Logger.Debug("jobAssignment = {0}", jobAssignment);
if (jobAssignment == null)
{
Logger.Debug("jobAssignment not found", jobAssignment);
response.StatusCode = (int)HttpStatusCode.NotFound;
response.StatusMessage = "No resource found on path '" + request.Path + "'.";
return;
}
var notification = request.QueryStringParameters;
Logger.Debug("notification = {0}", notification);
if (notification == null || !notification.Any())
{
response.StatusCode = (int)HttpStatusCode.BadRequest;
response.StatusMessage = "Missing notification in request Query String";
return;
}
// invoking worker lambda function that will process the notification
var lambdaClient = new AmazonLambdaClient();
var invokeRequest = new InvokeRequest
{
FunctionName = request.StageVariables["WorkerLambdaFunctionName"],
InvocationType = "Event",
LogType = "None",
Payload =
new
{
action = "ProcessNotification",
stageVariables = request.StageVariables,
jobAssignmentId,
notification
}.ToMcmaJson().ToString()
};
Logger.Debug("Invoking Lambda with payload: {0}", invokeRequest.Payload);
await lambdaClient.InvokeAsync(invokeRequest);
}
public static async Task CancelJobAsync(McmaApiRequestContext requestContext)
{
var table = new DynamoDbTable <Job>(requestContext.TableName());
var jobId = requestContext.PublicUrl() + "/jobs/" + requestContext.Request.PathVariables["id"];
if (!requestContext.ResourceIfFound(await table.GetAsync(jobId), false))
{
return;
}
requestContext.Response.StatusCode = (int)HttpStatusCode.NotImplemented;
requestContext.Response.StatusMessage = "Stopping job is not implemented";
}
public static async Task GetServiceAsync(ApiGatewayRequest request, McmaApiResponse response)
{
Logger.Debug(nameof(GetServiceAsync));
Logger.Debug(request.ToMcmaJson().ToString());
var table = new DynamoDbTable(request.StageVariables["TableName"]);
var serviceId = request.StageVariables["PublicUrl"] + request.Path;
response.JsonBody = (await table.GetAsync <Service>(serviceId))?.ToMcmaJson();
if (response.JsonBody == null)
{
response.StatusCode = (int)HttpStatusCode.NotFound;
response.StatusMessage = "No resource found on path '" + request.Path + "'.";
}
}
public static async Task CancelJobAsync(ApiGatewayRequest request, McmaApiResponse response)
{
var table = new DynamoDbTable(request.StageVariables["TableName"]);
var jobId = request.StageVariables["PublicUrl"] + "/jobs/" + request.PathVariables["id"];
var job = await table.GetAsync <Job>(jobId);
if (job == null)
{
response.StatusCode = (int)HttpStatusCode.NotFound;
response.StatusMessage = "No resource found on path '" + request.Path + "'";
return;
}
response.StatusCode = (int)HttpStatusCode.NotImplemented;
response.StatusMessage = "Canceling job is not implemented";
}
public static async Task GetJobAssignmentAsync(ApiGatewayRequest request, McmaApiResponse response)
{
Logger.Debug(nameof(GetJobAssignmentAsync));
Logger.Debug(request.ToMcmaJson().ToString());
var table = new DynamoDbTable(request.StageVariables["TableName"]);
var jobAssignmentId = request.StageVariables["PublicUrl"] + request.Path;
var jobAssignment = await table.GetAsync <JobAssignment>(jobAssignmentId);
response.JsonBody = jobAssignment != null?jobAssignment.ToMcmaJson() : null;
if (response.JsonBody == null)
{
response.StatusCode = (int)HttpStatusCode.NotFound;
response.StatusMessage = "No resource found on path '" + request.Path + "'.";
}
}
public static async Task ProcessNotificationAsync(McmaApiRequestContext requestContext)
{
Logger.Debug(nameof(ProcessNotificationAsync));
Logger.Debug(requestContext.Request.ToMcmaJson().ToString());
var table = new DynamoDbTable <JobAssignment>(requestContext.TableName());
var jobAssignmentId = requestContext.PublicUrl() + "/job-assignments/" + requestContext.Request.PathVariables["id"];
var jobAssignment = await table.GetAsync(jobAssignmentId);
Logger.Debug("jobAssignment = {0}", jobAssignment);
if (!requestContext.ResourceIfFound(jobAssignment, false))
{
return;
}
var notification = requestContext.Request.QueryStringParameters;
Logger.Debug("notification = {0}", notification);
if (notification == null || !notification.Any())
{
requestContext.Response.StatusCode = (int)HttpStatusCode.BadRequest;
requestContext.Response.StatusMessage = "Missing notification in request Query String";
return;
}
var lambdaWorkerInvoker = new LambdaWorkerInvoker();
await lambdaWorkerInvoker.RunAsync(
requestContext.WorkerFunctionName(),
new
{
operationName = "ProcessNotification",
contextVariables = requestContext.GetAllContextVariables(),
input = new
{
jobAssignmentId,
notification
}
});
}
public static async Task DeleteBmEssenceAsync(ApiGatewayRequest request, McmaApiResponse response)
{
Logger.Debug(nameof(DeleteBmEssenceAsync));
Logger.Debug(request.ToMcmaJson().ToString());
var table = new DynamoDbTable(request.StageVariables["TableName"]);
var bmEssenceId = request.StageVariables["PublicUrl"] + request.Path;
var bmEssence = await table.GetAsync <BMEssence>(bmEssenceId);
if (bmEssence == null)
{
response.StatusCode = (int)HttpStatusCode.NotFound;
response.StatusMessage = "No resource found on path '" + request.Path + "'.";
return;
}
await table.DeleteAsync <BMEssence>(bmEssenceId);
}
internal static async Task CreateJobProcessAsync(JobRepositoryWorkerRequest @event)
{
var jobId = @event.JobId;
var table = new DynamoDbTable(@event.Request.StageVariables["TableName"]);
var job = await table.GetAsync <Job>(jobId);
var resourceManager = @event.Request.GetAwsV4ResourceManager();
try
{
var jobProcess = new JobProcess {
Job = jobId, NotificationEndpoint = new NotificationEndpoint {
HttpEndpoint = jobId + "/notifications"
}
};
jobProcess = await resourceManager.CreateAsync(jobProcess);
job.Status = "QUEUED";
job.JobProcess = jobProcess.Id;
}
catch (Exception error)
{
Logger.Error("Failed to create JobProcess.");
Logger.Exception(error);
job.Status = "FAILED";
job.StatusMessage = $"Failed to create JobProcess due to error '{error}'";
}
job.DateModified = DateTime.UtcNow;
await table.PutAsync <Job>(jobId, job);
await resourceManager.SendNotificationAsync(job, job.NotificationEndpoint);
}
public static async Task CreateJobAssignmentAsync(WorkerRequest request, CreateJobAssignmentRequest createRequest)
{
var resourceManager = request.GetAwsV4ResourceManager();
var table = new DynamoDbTable <JobProcess>(request.TableName());
var jobProcessId = createRequest.JobProcessId;
var jobProcess = await table.GetAsync(jobProcessId);
try
{
// retrieving the job
var job = await resourceManager.ResolveAsync <Job>(jobProcess.Job);
// retrieving the jobProfile
var jobProfile = await resourceManager.ResolveAsync <JobProfile>(job.JobProfile);
// validating job.JobInput with required input parameters of jobProfile
var jobInput = job.JobInput; //await resourceManager.ResolveAsync<JobParameterBag>(job.JobInput);
if (jobInput == null)
{
throw new Exception("Job is missing jobInput");
}
if (jobProfile.InputParameters != null)
{
foreach (var parameter in jobProfile.InputParameters)
{
if (!jobInput.HasProperty(parameter.ParameterName))
{
throw new Exception("jobInput is missing required input parameter '" + parameter.ParameterName + "'");
}
}
}
// finding a service that is capable of handling the job type and job profile
var services = await resourceManager.GetAsync <Service>();
Service selectedService = null;
ResourceEndpointClient jobAssignmentResourceEndpoint = null;
foreach (var service in services)
{
var serviceClient = new ServiceClient(service, AwsEnvironment.GetDefaultAwsV4AuthProvider());
jobAssignmentResourceEndpoint = null;
if (service.JobType == job.Type)
{
jobAssignmentResourceEndpoint = serviceClient.GetResourceEndpoint <JobAssignment>();
if (jobAssignmentResourceEndpoint == null)
{
continue;
}
if (service.JobProfiles != null)
{
foreach (var serviceJobProfile in service.JobProfiles)
{
if (serviceJobProfile == job.JobProfile)
{
selectedService = service;
break;
}
}
}
}
if (selectedService != null)
{
break;
}
}
if (jobAssignmentResourceEndpoint == null)
{
throw new Exception("Failed to find service that could execute the " + job.GetType().Name);
}
var jobAssignment = new JobAssignment
{
Job = jobProcess.Job,
NotificationEndpoint = new NotificationEndpoint
{
HttpEndpoint = jobProcessId + "/notifications"
}
};
jobAssignment = await jobAssignmentResourceEndpoint.PostAsync <JobAssignment>(jobAssignment);
jobProcess.Status = "SCHEDULED";
jobProcess.JobAssignment = jobAssignment.Id;
}
catch (Exception error)
{
Logger.Error("Failed to create job assignment");
Logger.Exception(error);
jobProcess.Status = JobStatus.Failed;
jobProcess.StatusMessage = error.ToString();
}
jobProcess.DateModified = DateTime.UtcNow;
await table.PutAsync(jobProcessId, jobProcess);
await resourceManager.SendNotificationAsync(jobProcess, jobProcess.NotificationEndpoint);
}