Beispiel #1
0
 // Use this method to map all collections to their proper places.
 // Usage here is to confirm that the subsequent key is the same as the preceding key.
 // This forces that the REST call ensure all collection parameters are grouped together.
 // Ex. This is good: (key1=value&key1=value2)
 // Ex. This is bad: (key1=value&key2=value2)
 // Ex. This is bad: (key1=value&key2=value2&key1=value3)
 public static void MapToOptionWithCollection(EMRDeveloperOptions existingOptions, string key, string value, string lastKey)
 {
     if (key.Equals(lastKey))
     {
         if ("args".Equals(key))
         {
             existingOptions.Args.Add(value);
             return;
         }
         throw new ArgumentException(string.Format("The developer option '{0}' was not expected and is not understood.", key));
     }
     throw new ArgumentException(string.Format("The developer option '{0}' is grouped out of order in the REST call. Group collection parameters together in the request.", key));
 }
Beispiel #2
0
        private OperationResult EstablishClient(AddonManifest manifest, EMRDeveloperOptions devOptions, out IAmazonElasticMapReduce client)
        {
            OperationResult result;

            bool requireCreds;
            var  accessKey       = manifest.ProvisioningUsername;
            var  secretAccessKey = manifest.ProvisioningPassword;
            var  prop            = manifest.Properties.First(p => p.Key.Equals("requireDevCredentials", StringComparison.InvariantCultureIgnoreCase));

            //jobid = null;

            if (bool.TryParse(prop.Value, out requireCreds) && requireCreds)
            {
                if (!ValidateDevCreds(devOptions))
                {
                    client = null;
                    result = new OperationResult()
                    {
                        IsSuccess      = false,
                        EndUserMessage = "The add on requires that developer credentials are specified but none were provided."
                    };

                    return(result);
                }

                accessKey       = devOptions.AccessKey;
                secretAccessKey = devOptions.SecretAccessKey;
            }

            AWSCredentials credentials = new BasicAWSCredentials(accessKey, secretAccessKey);

            client = AWSClientFactory.CreateAmazonElasticMapReduceClient(credentials, RegionEndpoint.USEast1);

            //jobid = job.JobFlowId;

            result = new OperationResult()
            {
                IsSuccess = true
            };

            return(result);
        }
Beispiel #3
0
        public override OperationResult Deprovision(AddonDeprovisionRequest request)
        {
            var deprovisionResult = new ProvisionAddOnResult("")
            {
                IsSuccess = true
            };

            deprovisionResult.ConnectionData = "deprovision";
            AddonManifest manifest       = request.Manifest;
            string        connectionData = request.ConnectionData;
            string        devOptions     = request.DeveloperOptions;

            //string jobid = null;

            try
            {
                IAmazonElasticMapReduce client;
                //var conInfo = ConnectionInfo.Parse(connectionData);
                var developerOptions = EMRDeveloperOptions.Parse(devOptions);

                var establishClientResult = EstablishClient(manifest, developerOptions, out client);
                if (!establishClientResult.IsSuccess)
                {
                    deprovisionResult.EndUserMessage = establishClientResult.EndUserMessage;
                    return(deprovisionResult);
                }

                var result = client.TerminateJobFlows(new TerminateJobFlowsRequest()
                {
                    JobFlowIds = { connectionData }
                });

                deprovisionResult.IsSuccess      = true;
                deprovisionResult.EndUserMessage = "EMR Cluster Termination Request Successfully Invoked.";
            }
            catch (Exception)
            {
                deprovisionResult.EndUserMessage = "An error occurred during deprovisioning, please check the SOC logs for further assistance.";
            }

            return(deprovisionResult);
        }
Beispiel #4
0
        private OperationResult ParseDevOptions(string developerOptions, out EMRDeveloperOptions devOptions)
        {
            devOptions = null;
            var result = new OperationResult()
            {
                IsSuccess = false
            };
            var progress = "";

            try
            {
                progress  += "Parsing developer options...\n";
                devOptions = EMRDeveloperOptions.Parse(developerOptions);
            }
            catch (ArgumentException)
            {
                result.EndUserMessage = "Placeholder for ValidateDevCreds";
                return(result);
            }

            result.IsSuccess      = true;
            result.EndUserMessage = progress;
            return(result);
        }
Beispiel #5
0
        public static EMRDeveloperOptions Parse(string devOptions)
        {
            EMRDeveloperOptions options = new EMRDeveloperOptions();
            String lastKey = "";

            if (!string.IsNullOrWhiteSpace(devOptions))
            {
                // splitting all entries into arrays of optionPairs
                var optionPairs = devOptions.Split(new[] { '&' }, StringSplitOptions.RemoveEmptyEntries);
                foreach (var optionPair in optionPairs)
                {
                    // splitting all optionPairs into arrays of key/value denominations
                    var optionPairParts = optionPair.Split(new[] { '=' }, StringSplitOptions.RemoveEmptyEntries);
                    if (optionPairParts.Length == 2)
                    {
                        if (lastKey.Equals(optionPairParts[0].Trim().ToLowerInvariant()))
                        {
                            MapToOptionWithCollection(options, optionPairParts[0].Trim().ToLowerInvariant(), optionPairParts[1].Trim(), lastKey);
                        }
                        else
                        {
                            MapToOption(options, optionPairParts[0].Trim().ToLowerInvariant(), optionPairParts[1].Trim());
                            lastKey = optionPairParts[0].Trim().ToLowerInvariant();
                        }
                    }
                    else
                    {
                        throw new ArgumentException(
                                  string.Format(
                                      "Unable to parse developer options which should be in the form of 'option=value&nextOption=nextValue'. The option '{0}' was not properly constructed",
                                      optionPair));
                    }
                }
            }
            return(options);
        }
Beispiel #6
0
 private bool ValidateDevCreds(EMRDeveloperOptions devOptions)
 {
     return(!(string.IsNullOrWhiteSpace(devOptions.AccessKey) || string.IsNullOrWhiteSpace(devOptions.SecretAccessKey)));
 }
Beispiel #7
0
        public override ProvisionAddOnResult Provision(AddonProvisionRequest request)
        {
            var provisionResult = new ProvisionAddOnResult("")
            {
                IsSuccess = true
            };
            AddonManifest manifest         = request.Manifest;
            string        developerOptions = request.DeveloperOptions;

            try
            {
                IAmazonElasticMapReduce client;
                EMRDeveloperOptions     devOptions;

                var parseOptionsResult = ParseDevOptions(developerOptions, out devOptions);
                if (!parseOptionsResult.IsSuccess)
                {
                    provisionResult.EndUserMessage = parseOptionsResult.EndUserMessage;
                    return(provisionResult);
                }

                var establishClientResult = EstablishClient(manifest, EMRDeveloperOptions.Parse(developerOptions), out client);
                if (!establishClientResult.IsSuccess)
                {
                    provisionResult.EndUserMessage = establishClientResult.EndUserMessage;
                    return(provisionResult);
                }

                var stepFactory = new StepFactory();

                StepConfig enabledebugging = null;

                if (devOptions.EnableDebugging)
                {
                    enabledebugging = new StepConfig
                    {
                        Name            = "Enable debugging",
                        ActionOnFailure = "TERMINATE_JOB_FLOW",
                        HadoopJarStep   = stepFactory.NewEnableDebuggingStep()
                    };
                }

                var installHive = new StepConfig
                {
                    Name            = "Install Hive",
                    ActionOnFailure = "TERMINATE_JOB_FLOW",
                    HadoopJarStep   = stepFactory.NewInstallHiveStep()
                };

                var instanceConfig = new JobFlowInstancesConfig
                {
                    Ec2KeyName    = devOptions.Ec2KeyName,
                    HadoopVersion = "0.20",
                    InstanceCount = devOptions.InstanceCount,
                    // this is important. the EMR job flow must be kept alive for the application to see it during provisioning
                    KeepJobFlowAliveWhenNoSteps = true,
                    MasterInstanceType          = devOptions.MasterInstanceType,
                    SlaveInstanceType           = devOptions.SlaveInstanceType
                };

                var _request = new RunJobFlowRequest
                {
                    Name  = devOptions.JobFlowName,
                    Steps = { enabledebugging, installHive },
                    // revisit this one in ne
                    LogUri    = "s3://myawsbucket",
                    Instances = instanceConfig
                };

                // if debugging is enabled, add to top of the list of steps.
                if (devOptions.EnableDebugging)
                {
                    _request.Steps.Insert(0, enabledebugging);
                }

                var result = client.RunJobFlow(_request);

                // wait for JobFlowID to come back.
                while (result.JobFlowId == null)
                {
                    Thread.Sleep(1000);
                }

                provisionResult.IsSuccess      = true;
                provisionResult.ConnectionData = string.Format(result.JobFlowId);
            }
            catch (Exception e)
            {
                provisionResult.EndUserMessage = e.Message;
            }

            return(provisionResult);
        }
        // Interior method takes in instance of DeveloperOptions (aptly named existingOptions) and maps them to the proper value. In essence, a setter method.
        public static void MapToOption(EMRDeveloperOptions existingOptions, string key, string value)
        {
            if ("accesskey".Equals(key))
            {
                existingOptions.AccessKey = value;
                return;
            }

            if ("secretkey".Equals(key))
            {
                existingOptions.SecretAccessKey = value;
                return;
            }

            if ("availabilityzone".Equals(key))
            {
                existingOptions.AvailabilityZone = value;
                return;
            }

            if ("actiononfailure".Equals(key))
            {
                existingOptions.ActionOnFailure = new Amazon.ElasticMapReduce.ActionOnFailure(value);
                return;
            }

            if ("ec2keyname".Equals(key))
            {
                existingOptions.Ec2KeyName = value;
                return;
            }

            if("enabledebugging".Equals(key))
            {
                bool result;
                if(bool.TryParse(value, out result))
                {
                    throw new ArgumentException(string.Format("The developer option '{0}' can only have an integer value but '{1}' was used instead.", key, value));
                }
                existingOptions.EnableDebugging = result;
            }

            if ("instancecount".Equals(key))
            {
                int result;
                if (!int.TryParse(value, out result))
                {
                    throw new ArgumentException(string.Format("The developer option '{0}' can only have an integer value but '{1}' was used instead.", key, value));
                }
                existingOptions.InstanceCount = result;
                return;
            }

            if ("jar".Equals(key))
            {
                existingOptions.Jar = value;
                return;
            }

            if ("jobflowname".Equals(key))
            {
                existingOptions.JobFlowName = value;
                return;
            }

            if ("args".Equals(key))
            {
                existingOptions.Args.Add(value);
                return;
            }

            if ("loguri".Equals(key))
            {
                existingOptions.LogURI = value;
                return;
            }

            if ("mainclass".Equals(key))
            {
                existingOptions.MainClass = value;
                return;
            }

            if ("masterinstancetype".Equals(key))
            {
                existingOptions.MasterInstanceType = value;
                return;
            }

            if ("slaveinstancetype".Equals(key))
            {
                existingOptions.SlaveInstanceType = value;
                return;
            }

            if ("stepname".Equals(key))
            {
                existingOptions.stepName = value;
                return;
            }

            if ("keepjobflowalivewhennosteps".Equals(key))
            {
                bool result;
                if (!bool.TryParse(value, out result))
                {
                    throw new ArgumentException(string.Format("The developer option '{0}' must be a boolean value.", key));
                }
                existingOptions.KeepJobFlowaliveWhenNoSteps = result;
                return;
            }

            if ("secretaccesskey".Equals(key))
            {
                existingOptions.SecretAccessKey = value;
                return;
            }
            throw new ArgumentException(string.Format("The developer option '{0}' was not expected and is not understood.", key));
        }
        public static EMRDeveloperOptions Parse(string devOptions)
        {
            EMRDeveloperOptions options = new EMRDeveloperOptions();
            String lastKey = "";

            if (!string.IsNullOrWhiteSpace(devOptions))
            {
                // splitting all entries into arrays of optionPairs
                var optionPairs = devOptions.Split(new[] { '&' }, StringSplitOptions.RemoveEmptyEntries);
                foreach (var optionPair in optionPairs)
                {
                    // splitting all optionPairs into arrays of key/value denominations
                    var optionPairParts = optionPair.Split(new[] { '=' }, StringSplitOptions.RemoveEmptyEntries);
                    if (optionPairParts.Length == 2)
                    {
                        if (lastKey.Equals(optionPairParts[0].Trim().ToLowerInvariant()))
                        {
                            MapToOptionWithCollection(options, optionPairParts[0].Trim().ToLowerInvariant(), optionPairParts[1].Trim(), lastKey);
                        }
                        else
                        {
                            MapToOption(options, optionPairParts[0].Trim().ToLowerInvariant(), optionPairParts[1].Trim());
                            lastKey = optionPairParts[0].Trim().ToLowerInvariant();
                        }
                    }
                    else
                    {
                        throw new ArgumentException(
                            string.Format(
                                "Unable to parse developer options which should be in the form of 'option=value&nextOption=nextValue'. The option '{0}' was not properly constructed",
                                optionPair));
                    }
                }
            }
            return options;
        }
 // Use this method to map all collections to their proper places.
 // Usage here is to confirm that the subsequent key is the same as the preceding key.
 // This forces that the REST call ensure all collection parameters are grouped together.
 // Ex. This is good: (key1=value&key1=value2)
 // Ex. This is bad: (key1=value&key2=value2)
 // Ex. This is bad: (key1=value&key2=value2&key1=value3)
 public static void MapToOptionWithCollection(EMRDeveloperOptions existingOptions, string key, string value, string lastKey)
 {
     if (key.Equals(lastKey))
     {
         if ("args".Equals(key))
         {
             existingOptions.Args.Add(value);
             return;
         }
         throw new ArgumentException(string.Format("The developer option '{0}' was not expected and is not understood.", key));
     }
     throw new ArgumentException(string.Format("The developer option '{0}' is grouped out of order in the REST call. Group collection parameters together in the request.", key));
 }
 private bool ValidateDevCreds(EMRDeveloperOptions devOptions)
 {
     return !(string.IsNullOrWhiteSpace(devOptions.AccessKey) || string.IsNullOrWhiteSpace(devOptions.SecretAccessKey));
 }
        private OperationResult ParseDevOptions(string developerOptions, out EMRDeveloperOptions devOptions)
        {
            devOptions = null;
            var result = new OperationResult() { IsSuccess = false };
            var progress = "";

            try
            {
                progress += "Parsing developer options...\n";
                devOptions = EMRDeveloperOptions.Parse(developerOptions);
            }
            catch (ArgumentException)
            {
                result.EndUserMessage = "Placeholder for ValidateDevCreds";
                return result;
            }

            result.IsSuccess = true;
            result.EndUserMessage = progress;
            return result;
        }
        private OperationResult EstablishClient(AddonManifest manifest, EMRDeveloperOptions devOptions, out IAmazonElasticMapReduce client)
        {
            OperationResult result;

            bool requireCreds;
            var accessKey = manifest.ProvisioningUsername;
            var secretAccessKey = manifest.ProvisioningPassword;
            var prop = manifest.Properties.First(p => p.Key.Equals("requireDevCredentials", StringComparison.InvariantCultureIgnoreCase));
            //jobid = null;

            if (bool.TryParse(prop.Value, out requireCreds) && requireCreds)
            {
                if (!ValidateDevCreds(devOptions))
                {
                    client = null;
                    result = new OperationResult()
                    {
                        IsSuccess = false,
                        EndUserMessage = "The add on requires that developer credentials are specified but none were provided."
                    };

                    return result;
                }

                accessKey = devOptions.AccessKey;
                secretAccessKey = devOptions.SecretAccessKey;
            }

            AWSCredentials credentials = new BasicAWSCredentials(accessKey, secretAccessKey);
            client = AWSClientFactory.CreateAmazonElasticMapReduceClient(credentials, RegionEndpoint.USEast1);

            //jobid = job.JobFlowId;

            result = new OperationResult()
            {
                IsSuccess = true
            };

            return result;
        }
Beispiel #14
0
        // Interior method takes in instance of DeveloperOptions (aptly named existingOptions) and maps them to the proper value. In essence, a setter method.
        public static void MapToOption(EMRDeveloperOptions existingOptions, string key, string value)
        {
            if ("accesskey".Equals(key))
            {
                existingOptions.AccessKey = value;
                return;
            }

            if ("secretkey".Equals(key))
            {
                existingOptions.SecretAccessKey = value;
                return;
            }

            if ("availabilityzone".Equals(key))
            {
                existingOptions.AvailabilityZone = value;
                return;
            }

            if ("actiononfailure".Equals(key))
            {
                existingOptions.ActionOnFailure = new Amazon.ElasticMapReduce.ActionOnFailure(value);
                return;
            }

            if ("ec2keyname".Equals(key))
            {
                existingOptions.Ec2KeyName = value;
                return;
            }

            if ("enabledebugging".Equals(key))
            {
                bool result;
                if (bool.TryParse(value, out result))
                {
                    throw new ArgumentException(string.Format("The developer option '{0}' can only have an integer value but '{1}' was used instead.", key, value));
                }
                existingOptions.EnableDebugging = result;
            }


            if ("instancecount".Equals(key))
            {
                int result;
                if (!int.TryParse(value, out result))
                {
                    throw new ArgumentException(string.Format("The developer option '{0}' can only have an integer value but '{1}' was used instead.", key, value));
                }
                existingOptions.InstanceCount = result;
                return;
            }

            if ("jar".Equals(key))
            {
                existingOptions.Jar = value;
                return;
            }

            if ("jobflowname".Equals(key))
            {
                existingOptions.JobFlowName = value;
                return;
            }

            if ("args".Equals(key))
            {
                existingOptions.Args.Add(value);
                return;
            }

            if ("loguri".Equals(key))
            {
                existingOptions.LogURI = value;
                return;
            }

            if ("mainclass".Equals(key))
            {
                existingOptions.MainClass = value;
                return;
            }

            if ("masterinstancetype".Equals(key))
            {
                existingOptions.MasterInstanceType = value;
                return;
            }

            if ("slaveinstancetype".Equals(key))
            {
                existingOptions.SlaveInstanceType = value;
                return;
            }

            if ("stepname".Equals(key))
            {
                existingOptions.stepName = value;
                return;
            }

            if ("keepjobflowalivewhennosteps".Equals(key))
            {
                bool result;
                if (!bool.TryParse(value, out result))
                {
                    throw new ArgumentException(string.Format("The developer option '{0}' must be a boolean value.", key));
                }
                existingOptions.KeepJobFlowaliveWhenNoSteps = result;
                return;
            }

            if ("secretaccesskey".Equals(key))
            {
                existingOptions.SecretAccessKey = value;
                return;
            }
            throw new ArgumentException(string.Format("The developer option '{0}' was not expected and is not understood.", key));
        }