This class provides helper methods for creating common Elastic MapReduce step types. To use StepFactory, you should construct it with the appropriate bucket for your region. The official bucket format is "<region>.elasticmapreduce", so us-east-1 would use the bucket "us-east-1.elasticmapreduce".
StepFactory Class Documentation
Exemplo n.º 1
0
        public override ProvisionAddOnResult Provision(AddonProvisionRequest request)
        {
            var provisionResult = new ProvisionAddOnResult("") { IsSuccess = true };
            AddonManifest manifest = request.Manifest;
            string developerOptions = request.DeveloperOptions;

            try
            {
                IAmazonElasticMapReduce client;
                EMRDeveloperOptions devOptions;

                var parseOptionsResult = ParseDevOptions(developerOptions, out devOptions);
                if (!parseOptionsResult.IsSuccess)
                {
                    provisionResult.EndUserMessage = parseOptionsResult.EndUserMessage;
                    return provisionResult;
                }

                var establishClientResult = EstablishClient(manifest, EMRDeveloperOptions.Parse(developerOptions), out client);
                if (!establishClientResult.IsSuccess)
                {
                    provisionResult.EndUserMessage = establishClientResult.EndUserMessage;
                    return provisionResult;
                }

                var stepFactory = new StepFactory();

                StepConfig enabledebugging = null;

                if (devOptions.EnableDebugging)
                {
                    enabledebugging = new StepConfig
                    {
                        Name = "Enable debugging",
                        ActionOnFailure = "TERMINATE_JOB_FLOW",
                        HadoopJarStep = stepFactory.NewEnableDebuggingStep()
                    };
                }

                var installHive = new StepConfig
                {
                    Name = "Install Hive",
                    ActionOnFailure = "TERMINATE_JOB_FLOW",
                    HadoopJarStep = stepFactory.NewInstallHiveStep()
                };

                var instanceConfig = new JobFlowInstancesConfig
                {
                    Ec2KeyName = devOptions.Ec2KeyName,
                    HadoopVersion = "0.20",
                    InstanceCount = devOptions.InstanceCount,
                    // this is important. the EMR job flow must be kept alive for the application to see it during provisioning
                    KeepJobFlowAliveWhenNoSteps = true,
                    MasterInstanceType = devOptions.MasterInstanceType,
                    SlaveInstanceType = devOptions.SlaveInstanceType
                };

                var _request = new RunJobFlowRequest
                {
                    Name = devOptions.JobFlowName,
                    Steps = { enabledebugging, installHive },
                    // revisit this one in ne
                    LogUri = "s3://myawsbucket",
                    Instances = instanceConfig
                };

                // if debugging is enabled, add to top of the list of steps.
                if (devOptions.EnableDebugging)
                {
                    _request.Steps.Insert(0, enabledebugging);
                }

                var result = client.RunJobFlow(_request);

                // wait for JobFlowID to come back.
                while (result.JobFlowId == null)
                {
                    Thread.Sleep(1000);
                }

                provisionResult.IsSuccess = true;
                provisionResult.ConnectionData = string.Format(result.JobFlowId);
            }
            catch (Exception e)
            {
                provisionResult.EndUserMessage = e.Message;
            }

            return provisionResult;
        }