/// <summary> /// Process activity task /// </summary> /// <param name="input">Input for the activity task</param> /// <returns>Processing result: either an updated SWF activity, or an error message</returns> private async Task <ProcessSwfActivityResult> ProcessTask(string input) { ProcessSwfActivityResult result = new ProcessSwfActivityResult(); try { //read SWF activity info SwfActivity swfActivity = JsonSerializer.Deserialize <SwfActivity>(input); //based on SWF activity's info create an EMR activity SingleEmrActivityIterator singleEmrActivityIterator = new SingleEmrActivityIterator(swfActivity); //Run this activity on EMR Service using (EmrActivitiesRunner emrRunner = new EmrActivitiesRunner(this.EmrJobLogger, this.EmrJobStateChecker, this.EmrClient, this.Settings, singleEmrActivityIterator)) { emrRunner.JobFlowId = swfActivity.JobFlowId; //set JobFlowId for the current activity bool emrJobOk = await emrRunner.Start(); if (emrJobOk) { swfActivity.JobFlowId = emrRunner.JobFlowId; //read JobFlowId in case it was changed (for example, during starting a new EMR job) result.Output = swfActivity; } else { result.ErrorMessage = emrRunner.ErrorMessage; } } } catch (Exception ex) { result.ErrorMessage = ex.Message; } return(result); }
protected override IEnumerable <EmrActivity> GetNormalFlow(EmrActivitiesRunner emrRunner) { if (String.IsNullOrEmpty(emrRunner.JobFlowId)) { yield return(this.CreateStartActivity()); } yield return(this.CreateAddStepsActivity()); yield return(new TerminateJobActivity("Job succeeded. terminate cluster")); }
public async Task <bool> Run() { //Create dependencies IBuilderSettings settings = this.CreateSettings(); AmazonElasticMapReduceClient emrClient = this.CreateEmrClient(); IEmrJobLogger emrJobLogger = new EmrJobLogger(); IEmrJobStateChecker emrJobStateChecker = new EmrJobStateChecker(); DemoEmrActivitiesEnumerator activitiesIterator = new DemoEmrActivitiesEnumerator(); using (EmrActivitiesRunner emrRunner = new EmrActivitiesRunner(emrJobLogger, emrJobStateChecker, emrClient, settings, activitiesIterator)) { //explicitly set an existing jobFlowId, if you want to work with an existing job //emrRunner.JobFlowId = "j-36G3NHTVEP1Q7"; return(await emrRunner.Start()); } }
protected override IEnumerable <EmrActivity> GetNormalFlow(EmrActivitiesRunner emrRunner) { yield return(this.emrActivity); }
protected override IEnumerable <EmrActivity> GetFailedFlow(EmrActivitiesRunner emrRunner) { yield return(new TerminateJobActivity("Job failed. terminate cluster")); }