Specification of a job flow step.

Beispiel #1
0
 /// <summary>
 /// Instantiates StepDetail with the parameterized properties
 /// </summary>
 /// <param name="stepConfig">The step configuration.</param>
 /// <param name="executionStatusDetail">The description of the step status.</param>
 public StepDetail(StepConfig stepConfig, StepExecutionStatusDetail executionStatusDetail)
 {
     _stepConfig = stepConfig;
     _executionStatusDetail = executionStatusDetail;
 }
Beispiel #2
0
 /// <summary>
 /// Sets the StepConfig property
 /// </summary>
 /// <param name="stepConfig">The value to set for the StepConfig property </param>
 /// <returns>this instance</returns>
 public StepDetail WithStepConfig(StepConfig stepConfig)
 {
     this.stepConfig = stepConfig;
     return this;
 }
 /// <summary>
 /// Sets the StepConfig property
 /// </summary>
 /// <param name="stepConfig">The value to set for the StepConfig property </param>
 /// <returns>this instance</returns>
 public StepDetail WithStepConfig(StepConfig stepConfig)
 {
     this.stepConfig = stepConfig;
     return(this);
 }
Beispiel #4
0
 /// <summary>
 /// Instantiates StepDetail with the parameterized properties
 /// </summary>
 /// <param name="stepConfig">The step configuration.</param>
 /// <param name="executionStatusDetail">The description of the step status.</param>
 public StepDetail(StepConfig stepConfig, StepExecutionStatusDetail executionStatusDetail)
 {
     _stepConfig            = stepConfig;
     _executionStatusDetail = executionStatusDetail;
 }
        public override ProvisionAddOnResult Provision(AddonProvisionRequest request)
        {
            var provisionResult = new ProvisionAddOnResult("") { IsSuccess = true };
            AddonManifest manifest = request.Manifest;
            string developerOptions = request.DeveloperOptions;

            try
            {
                IAmazonElasticMapReduce client;
                EMRDeveloperOptions devOptions;

                var parseOptionsResult = ParseDevOptions(developerOptions, out devOptions);
                if (!parseOptionsResult.IsSuccess)
                {
                    provisionResult.EndUserMessage = parseOptionsResult.EndUserMessage;
                    return provisionResult;
                }

                var establishClientResult = EstablishClient(manifest, EMRDeveloperOptions.Parse(developerOptions), out client);
                if (!establishClientResult.IsSuccess)
                {
                    provisionResult.EndUserMessage = establishClientResult.EndUserMessage;
                    return provisionResult;
                }

                var stepFactory = new StepFactory();

                StepConfig enabledebugging = null;

                if (devOptions.EnableDebugging)
                {
                    enabledebugging = new StepConfig
                    {
                        Name = "Enable debugging",
                        ActionOnFailure = "TERMINATE_JOB_FLOW",
                        HadoopJarStep = stepFactory.NewEnableDebuggingStep()
                    };
                }

                var installHive = new StepConfig
                {
                    Name = "Install Hive",
                    ActionOnFailure = "TERMINATE_JOB_FLOW",
                    HadoopJarStep = stepFactory.NewInstallHiveStep()
                };

                var instanceConfig = new JobFlowInstancesConfig
                {
                    Ec2KeyName = devOptions.Ec2KeyName,
                    HadoopVersion = "0.20",
                    InstanceCount = devOptions.InstanceCount,
                    // this is important. the EMR job flow must be kept alive for the application to see it during provisioning
                    KeepJobFlowAliveWhenNoSteps = true,
                    MasterInstanceType = devOptions.MasterInstanceType,
                    SlaveInstanceType = devOptions.SlaveInstanceType
                };

                var _request = new RunJobFlowRequest
                {
                    Name = devOptions.JobFlowName,
                    Steps = { enabledebugging, installHive },
                    // revisit this one in ne
                    LogUri = "s3://myawsbucket",
                    Instances = instanceConfig
                };

                // if debugging is enabled, add to top of the list of steps.
                if (devOptions.EnableDebugging)
                {
                    _request.Steps.Insert(0, enabledebugging);
                }

                var result = client.RunJobFlow(_request);

                // wait for JobFlowID to come back.
                while (result.JobFlowId == null)
                {
                    Thread.Sleep(1000);
                }

                provisionResult.IsSuccess = true;
                provisionResult.ConnectionData = string.Format(result.JobFlowId);
            }
            catch (Exception e)
            {
                provisionResult.EndUserMessage = e.Message;
            }

            return provisionResult;
        }