Exemple #1
0
        public static async Task ProcessNotificationAsync(WorkerRequest request, ProcessNotificationRequest @event)
        {
            var jobProcessId        = @event.JobProcessId;
            var notification        = @event.Notification;
            var notificationJobData = notification.Content.ToMcmaObject <JobBase>();

            var table = new DynamoDbTable <JobProcess>(request.TableName());

            var jobProcess = await table.GetAsync(jobProcessId);

            // not updating job if it already was marked as completed or failed.
            if (jobProcess.Status == JobStatus.Completed || jobProcess.Status == JobStatus.Failed)
            {
                Logger.Warn("Ignoring update of job process that tried to change state from " + jobProcess.Status + " to " + notificationJobData.Status);
                return;
            }

            jobProcess.Status        = notificationJobData.Status;
            jobProcess.StatusMessage = notificationJobData.StatusMessage;
            jobProcess.Progress      = notificationJobData.Progress;
            jobProcess.JobOutput     = notificationJobData.JobOutput;
            jobProcess.DateModified  = DateTime.UtcNow;

            await table.PutAsync(jobProcessId, jobProcess);

            var resourceManager = request.GetAwsV4ResourceManager();

            await resourceManager.SendNotificationAsync(jobProcess, jobProcess.NotificationEndpoint);
        }
        internal static async Task ProcessNotificationAsync(WorkflowServiceWorkerRequest @event)
        {
            var jobAssignmentId        = @event.JobAssignmentId;
            var notification           = @event.Notification;
            var notificationJobPayload = notification.Content.ToMcmaObject <JobBase>();

            var table = new DynamoDbTable(@event.Request.StageVariables["TableName"]);

            var jobAssignment = await table.GetAsync <JobAssignment>(jobAssignmentId);

            jobAssignment.Status        = notificationJobPayload.Status;
            jobAssignment.StatusMessage = notificationJobPayload.StatusMessage;
            if (notificationJobPayload.Progress != null)
            {
                jobAssignment.Progress = notificationJobPayload.Progress;
            }

            jobAssignment.JobOutput    = notificationJobPayload.JobOutput;
            jobAssignment.DateModified = DateTime.UtcNow;

            await table.PutAsync <JobAssignment>(jobAssignmentId, jobAssignment);

            var resourceManager = @event.Request.GetAwsV4ResourceManager();

            await resourceManager.SendNotificationAsync(jobAssignment, jobAssignment.NotificationEndpoint);
        }
        public static async Task ProcessNotificationAsync(McmaApiRequestContext requestContext)
        {
            var table = new DynamoDbTable <JobAssignment>(requestContext.TableName());

            var jobAssignment =
                await table.GetAsync(requestContext.PublicUrl() + "/job-assignments/" + requestContext.Request.PathVariables["id"]);

            if (!requestContext.ResourceIfFound(jobAssignment, false) || requestContext.IsBadRequestDueToMissingBody(out Notification notification))
            {
                return;
            }

            await WorkerInvoker.RunAsync(
                requestContext.WorkerFunctionName(),
                new
            {
                operationName    = "ProcessNotification",
                contextVariables = requestContext.GetAllContextVariables(),
                input            = new
                {
                    jobAssignmentId = jobAssignment.Id,
                    notification    = notification
                }
            });
        }
        public static async Task DeleteJobAsync(ApiGatewayRequest request, McmaApiResponse response)
        {
            Logger.Debug(nameof(DeleteJobAsync));
            Logger.Debug(request.ToMcmaJson().ToString());

            var table = new DynamoDbTable(request.StageVariables["TableName"]);

            var jobId = request.StageVariables["PublicUrl"] + request.Path;

            var job = await table.GetAsync <Job>(jobId);

            if (job == null)
            {
                response.StatusCode    = (int)HttpStatusCode.NotFound;
                response.StatusMessage = "No resource found on path '" + request.Path + "'.";
                return;
            }

            await table.DeleteAsync <Job>(jobId);

            // invoking worker lambda function that will delete the JobProcess created for this Job
            if (!string.IsNullOrEmpty(job.JobProcess))
            {
                var lambdaClient  = new AmazonLambdaClient();
                var invokeRequest = new InvokeRequest
                {
                    FunctionName   = request.StageVariables["WorkerLambdaFunctionName"],
                    InvocationType = "Event",
                    LogType        = "None",
                    Payload        = new { action = "deleteJobProcess", request = request, jobProcessId = job.JobProcess }.ToMcmaJson().ToString()
                };

                await lambdaClient.InvokeAsync(invokeRequest);
            }
        }
Exemple #5
0
        public static async Task ProcessNotificationAsync(McmaApiRequestContext requestContext)
        {
            var request  = requestContext.Request;
            var response = requestContext.Response;

            var table = new DynamoDbTable <JobProcess>(requestContext.TableName());

            var jobProcess = await table.GetAsync(requestContext.PublicUrl() + "/job-processes/" + request.PathVariables["id"]);

            if (!requestContext.ResourceIfFound(jobProcess, false) ||
                requestContext.IsBadRequestDueToMissingBody <Notification>(out var notification))
            {
                return;
            }

            if (jobProcess.JobAssignment != notification.Source)
            {
                response.StatusCode    = (int)HttpStatusCode.BadRequest;
                response.StatusMessage = "Unexpected notification from '" + notification.Source + "'.";
                return;
            }

            await WorkerInvoker.RunAsync(
                requestContext.WorkerFunctionName(),
                new
            {
                operationName    = "processNotification",
                contextVariables = requestContext.GetAllContextVariables(),
                input            = new
                {
                    jobProcessId = jobProcess.Id,
                    notification = notification
                }
            });
        }
        internal static async Task ProcessNotificationAsync(JobRepositoryWorkerRequest @event)
        {
            var jobId           = @event.JobId;
            var notification    = @event.Notification;
            var notificationJob = notification.Content.ToMcmaObject <JobBase>();

            var table = new DynamoDbTable(@event.Request.StageVariables["TableName"]);

            var job = await table.GetAsync <Job>(jobId);

            // not updating job if it already was marked as completed or failed.
            if (job.Status == "COMPLETED" || job.Status == "FAILED")
            {
                Logger.Warn("Ignoring update of job that tried to change state from " + job.Status + " to " + notificationJob.Status);
                return;
            }

            job.Status        = notificationJob.Status;
            job.StatusMessage = notificationJob.StatusMessage;
            job.Progress      = notificationJob.Progress;
            job.JobOutput     = notificationJob.JobOutput;
            job.DateModified  = DateTime.UtcNow;

            await table.PutAsync <Job>(jobId, job);

            var resourceManager = @event.Request.GetAwsV4ResourceManager();

            await resourceManager.SendNotificationAsync(job, job.NotificationEndpoint);
        }
Exemple #7
0
        protected override async Task ExecuteAsync(WorkerRequest @event, ProcessNotificationRequest notificationRequest)
        {
            var jobAssignmentId        = notificationRequest.JobAssignmentId;
            var notification           = notificationRequest.Notification;
            var notificationJobPayload = notification.Content.ToMcmaObject <JobBase>();

            var table = new DynamoDbTable <JobAssignment>(@event.TableName());

            var jobAssignment = await table.GetAsync(jobAssignmentId);

            jobAssignment.Status        = notificationJobPayload.Status;
            jobAssignment.StatusMessage = notificationJobPayload.StatusMessage;
            if (notificationJobPayload.Progress != null)
            {
                jobAssignment.Progress = notificationJobPayload.Progress;
            }

            jobAssignment.JobOutput    = notificationJobPayload.JobOutput;
            jobAssignment.DateModified = DateTime.UtcNow;

            await table.PutAsync(jobAssignmentId, jobAssignment);

            var resourceManager = @event.GetAwsV4ResourceManager();

            await resourceManager.SendNotificationAsync(jobAssignment, jobAssignment.NotificationEndpoint);
        }
        public static async Task ProcessNotificationAsync(ApiGatewayRequest request, McmaApiResponse response)
        {
            var table = new DynamoDbTable(request.StageVariables["TableName"]);

            var jobAssignmentId = request.StageVariables["PublicUrl"] + "/job-assignments/" + request.PathVariables["id"];

            var jobAssignment = await table.GetAsync <JobAssignment>(jobAssignmentId);

            if (jobAssignment == null)
            {
                response.StatusCode    = (int)HttpStatusCode.NotFound;
                response.StatusMessage = "No resource found on path '" + request.Path + "'";
                return;
            }

            var notification = request.JsonBody?.ToMcmaObject <Notification>();

            if (notification == null)
            {
                response.StatusCode    = (int)HttpStatusCode.BadRequest;
                response.StatusMessage = "Missing notification in request body";
                return;
            }

            var lambdaClient  = new AmazonLambdaClient();
            var invokeRequest = new InvokeRequest
            {
                FunctionName   = request.StageVariables["WorkerLambdaFunctionName"],
                InvocationType = "Event",
                LogType        = "None",
                Payload        = new { action = "ProcessNotification", request = request, jobAssignmentId = jobAssignmentId, notification = notification }.ToMcmaJson().ToString()
            };

            await lambdaClient.InvokeAsync(invokeRequest);
        }
Exemple #9
0
        private static async Task <JobAssignment> GetJobAssignmentAsync(DynamoDbTable table, string jobAssignmentId)
        {
            var jobAssignment = await table.GetAsync <JobAssignment>(jobAssignmentId);

            if (jobAssignment == null)
            {
                throw new Exception("JobAssignment with id '" + jobAssignmentId + "' not found");
            }
            return(jobAssignment);
        }
Exemple #10
0
        public static async Task ProcessNotificationAsync(ApiGatewayRequest request, McmaApiResponse response)
        {
            Logger.Debug(nameof(ProcessNotificationAsync));
            Logger.Debug(request.ToMcmaJson().ToString());

            var table = new DynamoDbTable(request.StageVariables["TableName"]);

            var jobAssignmentId = request.StageVariables["PublicUrl"] + "/job-assignments/" + request.PathVariables["id"];

            var jobAssignment = await table.GetAsync <JobAssignment>(jobAssignmentId);

            Logger.Debug("jobAssignment = {0}", jobAssignment);

            if (jobAssignment == null)
            {
                Logger.Debug("jobAssignment not found", jobAssignment);
                response.StatusCode    = (int)HttpStatusCode.NotFound;
                response.StatusMessage = "No resource found on path '" + request.Path + "'.";
                return;
            }

            var notification = request.QueryStringParameters;

            Logger.Debug("notification = {0}", notification);
            if (notification == null || !notification.Any())
            {
                response.StatusCode    = (int)HttpStatusCode.BadRequest;
                response.StatusMessage = "Missing notification in request Query String";
                return;
            }

            // invoking worker lambda function that will process the notification
            var lambdaClient  = new AmazonLambdaClient();
            var invokeRequest = new InvokeRequest
            {
                FunctionName   = request.StageVariables["WorkerLambdaFunctionName"],
                InvocationType = "Event",
                LogType        = "None",
                Payload        =
                    new
                {
                    action         = "ProcessNotification",
                    stageVariables = request.StageVariables,
                    jobAssignmentId,
                    notification
                }.ToMcmaJson().ToString()
            };

            Logger.Debug("Invoking Lambda with payload: {0}", invokeRequest.Payload);

            await lambdaClient.InvokeAsync(invokeRequest);
        }
Exemple #11
0
        public static async Task CancelJobAsync(McmaApiRequestContext requestContext)
        {
            var table = new DynamoDbTable <Job>(requestContext.TableName());

            var jobId = requestContext.PublicUrl() + "/jobs/" + requestContext.Request.PathVariables["id"];

            if (!requestContext.ResourceIfFound(await table.GetAsync(jobId), false))
            {
                return;
            }

            requestContext.Response.StatusCode    = (int)HttpStatusCode.NotImplemented;
            requestContext.Response.StatusMessage = "Stopping job is not implemented";
        }
        public static async Task GetServiceAsync(ApiGatewayRequest request, McmaApiResponse response)
        {
            Logger.Debug(nameof(GetServiceAsync));
            Logger.Debug(request.ToMcmaJson().ToString());

            var table = new DynamoDbTable(request.StageVariables["TableName"]);

            var serviceId = request.StageVariables["PublicUrl"] + request.Path;

            response.JsonBody = (await table.GetAsync <Service>(serviceId))?.ToMcmaJson();

            if (response.JsonBody == null)
            {
                response.StatusCode    = (int)HttpStatusCode.NotFound;
                response.StatusMessage = "No resource found on path '" + request.Path + "'.";
            }
        }
        public static async Task CancelJobAsync(ApiGatewayRequest request, McmaApiResponse response)
        {
            var table = new DynamoDbTable(request.StageVariables["TableName"]);

            var jobId = request.StageVariables["PublicUrl"] + "/jobs/" + request.PathVariables["id"];

            var job = await table.GetAsync <Job>(jobId);

            if (job == null)
            {
                response.StatusCode    = (int)HttpStatusCode.NotFound;
                response.StatusMessage = "No resource found on path '" + request.Path + "'";
                return;
            }

            response.StatusCode    = (int)HttpStatusCode.NotImplemented;
            response.StatusMessage = "Canceling job is not implemented";
        }
Exemple #14
0
        public static async Task GetJobAssignmentAsync(ApiGatewayRequest request, McmaApiResponse response)
        {
            Logger.Debug(nameof(GetJobAssignmentAsync));
            Logger.Debug(request.ToMcmaJson().ToString());

            var table = new DynamoDbTable(request.StageVariables["TableName"]);

            var jobAssignmentId = request.StageVariables["PublicUrl"] + request.Path;

            var jobAssignment = await table.GetAsync <JobAssignment>(jobAssignmentId);

            response.JsonBody = jobAssignment != null?jobAssignment.ToMcmaJson() : null;

            if (response.JsonBody == null)
            {
                response.StatusCode    = (int)HttpStatusCode.NotFound;
                response.StatusMessage = "No resource found on path '" + request.Path + "'.";
            }
        }
Exemple #15
0
        public static async Task ProcessNotificationAsync(McmaApiRequestContext requestContext)
        {
            Logger.Debug(nameof(ProcessNotificationAsync));
            Logger.Debug(requestContext.Request.ToMcmaJson().ToString());

            var table = new DynamoDbTable <JobAssignment>(requestContext.TableName());

            var jobAssignmentId = requestContext.PublicUrl() + "/job-assignments/" + requestContext.Request.PathVariables["id"];

            var jobAssignment = await table.GetAsync(jobAssignmentId);

            Logger.Debug("jobAssignment = {0}", jobAssignment);

            if (!requestContext.ResourceIfFound(jobAssignment, false))
            {
                return;
            }

            var notification = requestContext.Request.QueryStringParameters;

            Logger.Debug("notification = {0}", notification);
            if (notification == null || !notification.Any())
            {
                requestContext.Response.StatusCode    = (int)HttpStatusCode.BadRequest;
                requestContext.Response.StatusMessage = "Missing notification in request Query String";
                return;
            }

            var lambdaWorkerInvoker = new LambdaWorkerInvoker();
            await lambdaWorkerInvoker.RunAsync(
                requestContext.WorkerFunctionName(),
                new
            {
                operationName    = "ProcessNotification",
                contextVariables = requestContext.GetAllContextVariables(),
                input            = new
                {
                    jobAssignmentId,
                    notification
                }
            });
        }
        public static async Task DeleteBmEssenceAsync(ApiGatewayRequest request, McmaApiResponse response)
        {
            Logger.Debug(nameof(DeleteBmEssenceAsync));
            Logger.Debug(request.ToMcmaJson().ToString());

            var table = new DynamoDbTable(request.StageVariables["TableName"]);

            var bmEssenceId = request.StageVariables["PublicUrl"] + request.Path;

            var bmEssence = await table.GetAsync <BMEssence>(bmEssenceId);

            if (bmEssence == null)
            {
                response.StatusCode    = (int)HttpStatusCode.NotFound;
                response.StatusMessage = "No resource found on path '" + request.Path + "'.";
                return;
            }

            await table.DeleteAsync <BMEssence>(bmEssenceId);
        }
        internal static async Task CreateJobProcessAsync(JobRepositoryWorkerRequest @event)
        {
            var jobId = @event.JobId;

            var table = new DynamoDbTable(@event.Request.StageVariables["TableName"]);
            var job   = await table.GetAsync <Job>(jobId);

            var resourceManager = @event.Request.GetAwsV4ResourceManager();

            try
            {
                var jobProcess = new JobProcess {
                    Job = jobId, NotificationEndpoint = new NotificationEndpoint {
                        HttpEndpoint = jobId + "/notifications"
                    }
                };
                jobProcess = await resourceManager.CreateAsync(jobProcess);

                job.Status     = "QUEUED";
                job.JobProcess = jobProcess.Id;
            }
            catch (Exception error)
            {
                Logger.Error("Failed to create JobProcess.");
                Logger.Exception(error);

                job.Status        = "FAILED";
                job.StatusMessage = $"Failed to create JobProcess due to error '{error}'";
            }

            job.DateModified = DateTime.UtcNow;

            await table.PutAsync <Job>(jobId, job);

            await resourceManager.SendNotificationAsync(job, job.NotificationEndpoint);
        }
Exemple #18
0
        public static async Task CreateJobAssignmentAsync(WorkerRequest request, CreateJobAssignmentRequest createRequest)
        {
            var resourceManager = request.GetAwsV4ResourceManager();

            var table = new DynamoDbTable <JobProcess>(request.TableName());

            var jobProcessId = createRequest.JobProcessId;
            var jobProcess   = await table.GetAsync(jobProcessId);

            try
            {
                // retrieving the job
                var job = await resourceManager.ResolveAsync <Job>(jobProcess.Job);

                // retrieving the jobProfile
                var jobProfile = await resourceManager.ResolveAsync <JobProfile>(job.JobProfile);

                // validating job.JobInput with required input parameters of jobProfile
                var jobInput = job.JobInput; //await resourceManager.ResolveAsync<JobParameterBag>(job.JobInput);
                if (jobInput == null)
                {
                    throw new Exception("Job is missing jobInput");
                }

                if (jobProfile.InputParameters != null)
                {
                    foreach (var parameter in jobProfile.InputParameters)
                    {
                        if (!jobInput.HasProperty(parameter.ParameterName))
                        {
                            throw new Exception("jobInput is missing required input parameter '" + parameter.ParameterName + "'");
                        }
                    }
                }

                // finding a service that is capable of handling the job type and job profile
                var services = await resourceManager.GetAsync <Service>();

                Service selectedService = null;
                ResourceEndpointClient jobAssignmentResourceEndpoint = null;

                foreach (var service in services)
                {
                    var serviceClient = new ServiceClient(service, AwsEnvironment.GetDefaultAwsV4AuthProvider());

                    jobAssignmentResourceEndpoint = null;

                    if (service.JobType == job.Type)
                    {
                        jobAssignmentResourceEndpoint = serviceClient.GetResourceEndpoint <JobAssignment>();

                        if (jobAssignmentResourceEndpoint == null)
                        {
                            continue;
                        }

                        if (service.JobProfiles != null)
                        {
                            foreach (var serviceJobProfile in service.JobProfiles)
                            {
                                if (serviceJobProfile == job.JobProfile)
                                {
                                    selectedService = service;
                                    break;
                                }
                            }
                        }
                    }

                    if (selectedService != null)
                    {
                        break;
                    }
                }

                if (jobAssignmentResourceEndpoint == null)
                {
                    throw new Exception("Failed to find service that could execute the " + job.GetType().Name);
                }

                var jobAssignment = new JobAssignment
                {
                    Job = jobProcess.Job,
                    NotificationEndpoint = new NotificationEndpoint
                    {
                        HttpEndpoint = jobProcessId + "/notifications"
                    }
                };

                jobAssignment = await jobAssignmentResourceEndpoint.PostAsync <JobAssignment>(jobAssignment);

                jobProcess.Status        = "SCHEDULED";
                jobProcess.JobAssignment = jobAssignment.Id;
            }
            catch (Exception error)
            {
                Logger.Error("Failed to create job assignment");
                Logger.Exception(error);

                jobProcess.Status        = JobStatus.Failed;
                jobProcess.StatusMessage = error.ToString();
            }

            jobProcess.DateModified = DateTime.UtcNow;

            await table.PutAsync(jobProcessId, jobProcess);

            await resourceManager.SendNotificationAsync(jobProcess, jobProcess.NotificationEndpoint);
        }