// Use this method to map all collections to their proper places. // Usage here is to confirm that the subsequent key is the same as the preceding key. // This forces that the REST call ensure all collection parameters are grouped together. // Ex. This is good: (key1=value&key1=value2) // Ex. This is bad: (key1=value&key2=value2) // Ex. This is bad: (key1=value&key2=value2&key1=value3) public static void MapToOptionWithCollection(EMRDeveloperOptions existingOptions, string key, string value, string lastKey) { if (key.Equals(lastKey)) { if ("args".Equals(key)) { existingOptions.Args.Add(value); return; } throw new ArgumentException(string.Format("The developer option '{0}' was not expected and is not understood.", key)); } throw new ArgumentException(string.Format("The developer option '{0}' is grouped out of order in the REST call. Group collection parameters together in the request.", key)); }
private OperationResult EstablishClient(AddonManifest manifest, EMRDeveloperOptions devOptions, out IAmazonElasticMapReduce client) { OperationResult result; bool requireCreds; var accessKey = manifest.ProvisioningUsername; var secretAccessKey = manifest.ProvisioningPassword; var prop = manifest.Properties.First(p => p.Key.Equals("requireDevCredentials", StringComparison.InvariantCultureIgnoreCase)); //jobid = null; if (bool.TryParse(prop.Value, out requireCreds) && requireCreds) { if (!ValidateDevCreds(devOptions)) { client = null; result = new OperationResult() { IsSuccess = false, EndUserMessage = "The add on requires that developer credentials are specified but none were provided." }; return(result); } accessKey = devOptions.AccessKey; secretAccessKey = devOptions.SecretAccessKey; } AWSCredentials credentials = new BasicAWSCredentials(accessKey, secretAccessKey); client = AWSClientFactory.CreateAmazonElasticMapReduceClient(credentials, RegionEndpoint.USEast1); //jobid = job.JobFlowId; result = new OperationResult() { IsSuccess = true }; return(result); }
public override OperationResult Deprovision(AddonDeprovisionRequest request) { var deprovisionResult = new ProvisionAddOnResult("") { IsSuccess = true }; deprovisionResult.ConnectionData = "deprovision"; AddonManifest manifest = request.Manifest; string connectionData = request.ConnectionData; string devOptions = request.DeveloperOptions; //string jobid = null; try { IAmazonElasticMapReduce client; //var conInfo = ConnectionInfo.Parse(connectionData); var developerOptions = EMRDeveloperOptions.Parse(devOptions); var establishClientResult = EstablishClient(manifest, developerOptions, out client); if (!establishClientResult.IsSuccess) { deprovisionResult.EndUserMessage = establishClientResult.EndUserMessage; return(deprovisionResult); } var result = client.TerminateJobFlows(new TerminateJobFlowsRequest() { JobFlowIds = { connectionData } }); deprovisionResult.IsSuccess = true; deprovisionResult.EndUserMessage = "EMR Cluster Termination Request Successfully Invoked."; } catch (Exception) { deprovisionResult.EndUserMessage = "An error occurred during deprovisioning, please check the SOC logs for further assistance."; } return(deprovisionResult); }
private OperationResult ParseDevOptions(string developerOptions, out EMRDeveloperOptions devOptions) { devOptions = null; var result = new OperationResult() { IsSuccess = false }; var progress = ""; try { progress += "Parsing developer options...\n"; devOptions = EMRDeveloperOptions.Parse(developerOptions); } catch (ArgumentException) { result.EndUserMessage = "Placeholder for ValidateDevCreds"; return(result); } result.IsSuccess = true; result.EndUserMessage = progress; return(result); }
public static EMRDeveloperOptions Parse(string devOptions) { EMRDeveloperOptions options = new EMRDeveloperOptions(); String lastKey = ""; if (!string.IsNullOrWhiteSpace(devOptions)) { // splitting all entries into arrays of optionPairs var optionPairs = devOptions.Split(new[] { '&' }, StringSplitOptions.RemoveEmptyEntries); foreach (var optionPair in optionPairs) { // splitting all optionPairs into arrays of key/value denominations var optionPairParts = optionPair.Split(new[] { '=' }, StringSplitOptions.RemoveEmptyEntries); if (optionPairParts.Length == 2) { if (lastKey.Equals(optionPairParts[0].Trim().ToLowerInvariant())) { MapToOptionWithCollection(options, optionPairParts[0].Trim().ToLowerInvariant(), optionPairParts[1].Trim(), lastKey); } else { MapToOption(options, optionPairParts[0].Trim().ToLowerInvariant(), optionPairParts[1].Trim()); lastKey = optionPairParts[0].Trim().ToLowerInvariant(); } } else { throw new ArgumentException( string.Format( "Unable to parse developer options which should be in the form of 'option=value&nextOption=nextValue'. The option '{0}' was not properly constructed", optionPair)); } } } return(options); }
private bool ValidateDevCreds(EMRDeveloperOptions devOptions) { return(!(string.IsNullOrWhiteSpace(devOptions.AccessKey) || string.IsNullOrWhiteSpace(devOptions.SecretAccessKey))); }
public override ProvisionAddOnResult Provision(AddonProvisionRequest request) { var provisionResult = new ProvisionAddOnResult("") { IsSuccess = true }; AddonManifest manifest = request.Manifest; string developerOptions = request.DeveloperOptions; try { IAmazonElasticMapReduce client; EMRDeveloperOptions devOptions; var parseOptionsResult = ParseDevOptions(developerOptions, out devOptions); if (!parseOptionsResult.IsSuccess) { provisionResult.EndUserMessage = parseOptionsResult.EndUserMessage; return(provisionResult); } var establishClientResult = EstablishClient(manifest, EMRDeveloperOptions.Parse(developerOptions), out client); if (!establishClientResult.IsSuccess) { provisionResult.EndUserMessage = establishClientResult.EndUserMessage; return(provisionResult); } var stepFactory = new StepFactory(); StepConfig enabledebugging = null; if (devOptions.EnableDebugging) { enabledebugging = new StepConfig { Name = "Enable debugging", ActionOnFailure = "TERMINATE_JOB_FLOW", HadoopJarStep = stepFactory.NewEnableDebuggingStep() }; } var installHive = new StepConfig { Name = "Install Hive", ActionOnFailure = "TERMINATE_JOB_FLOW", HadoopJarStep = stepFactory.NewInstallHiveStep() }; var instanceConfig = new JobFlowInstancesConfig { Ec2KeyName = devOptions.Ec2KeyName, HadoopVersion = "0.20", InstanceCount = devOptions.InstanceCount, // this is important. the EMR job flow must be kept alive for the application to see it during provisioning KeepJobFlowAliveWhenNoSteps = true, MasterInstanceType = devOptions.MasterInstanceType, SlaveInstanceType = devOptions.SlaveInstanceType }; var _request = new RunJobFlowRequest { Name = devOptions.JobFlowName, Steps = { enabledebugging, installHive }, // revisit this one in ne LogUri = "s3://myawsbucket", Instances = instanceConfig }; // if debugging is enabled, add to top of the list of steps. if (devOptions.EnableDebugging) { _request.Steps.Insert(0, enabledebugging); } var result = client.RunJobFlow(_request); // wait for JobFlowID to come back. while (result.JobFlowId == null) { Thread.Sleep(1000); } provisionResult.IsSuccess = true; provisionResult.ConnectionData = string.Format(result.JobFlowId); } catch (Exception e) { provisionResult.EndUserMessage = e.Message; } return(provisionResult); }
// Interior method takes in instance of DeveloperOptions (aptly named existingOptions) and maps them to the proper value. In essence, a setter method. public static void MapToOption(EMRDeveloperOptions existingOptions, string key, string value) { if ("accesskey".Equals(key)) { existingOptions.AccessKey = value; return; } if ("secretkey".Equals(key)) { existingOptions.SecretAccessKey = value; return; } if ("availabilityzone".Equals(key)) { existingOptions.AvailabilityZone = value; return; } if ("actiononfailure".Equals(key)) { existingOptions.ActionOnFailure = new Amazon.ElasticMapReduce.ActionOnFailure(value); return; } if ("ec2keyname".Equals(key)) { existingOptions.Ec2KeyName = value; return; } if("enabledebugging".Equals(key)) { bool result; if(bool.TryParse(value, out result)) { throw new ArgumentException(string.Format("The developer option '{0}' can only have an integer value but '{1}' was used instead.", key, value)); } existingOptions.EnableDebugging = result; } if ("instancecount".Equals(key)) { int result; if (!int.TryParse(value, out result)) { throw new ArgumentException(string.Format("The developer option '{0}' can only have an integer value but '{1}' was used instead.", key, value)); } existingOptions.InstanceCount = result; return; } if ("jar".Equals(key)) { existingOptions.Jar = value; return; } if ("jobflowname".Equals(key)) { existingOptions.JobFlowName = value; return; } if ("args".Equals(key)) { existingOptions.Args.Add(value); return; } if ("loguri".Equals(key)) { existingOptions.LogURI = value; return; } if ("mainclass".Equals(key)) { existingOptions.MainClass = value; return; } if ("masterinstancetype".Equals(key)) { existingOptions.MasterInstanceType = value; return; } if ("slaveinstancetype".Equals(key)) { existingOptions.SlaveInstanceType = value; return; } if ("stepname".Equals(key)) { existingOptions.stepName = value; return; } if ("keepjobflowalivewhennosteps".Equals(key)) { bool result; if (!bool.TryParse(value, out result)) { throw new ArgumentException(string.Format("The developer option '{0}' must be a boolean value.", key)); } existingOptions.KeepJobFlowaliveWhenNoSteps = result; return; } if ("secretaccesskey".Equals(key)) { existingOptions.SecretAccessKey = value; return; } throw new ArgumentException(string.Format("The developer option '{0}' was not expected and is not understood.", key)); }
public static EMRDeveloperOptions Parse(string devOptions) { EMRDeveloperOptions options = new EMRDeveloperOptions(); String lastKey = ""; if (!string.IsNullOrWhiteSpace(devOptions)) { // splitting all entries into arrays of optionPairs var optionPairs = devOptions.Split(new[] { '&' }, StringSplitOptions.RemoveEmptyEntries); foreach (var optionPair in optionPairs) { // splitting all optionPairs into arrays of key/value denominations var optionPairParts = optionPair.Split(new[] { '=' }, StringSplitOptions.RemoveEmptyEntries); if (optionPairParts.Length == 2) { if (lastKey.Equals(optionPairParts[0].Trim().ToLowerInvariant())) { MapToOptionWithCollection(options, optionPairParts[0].Trim().ToLowerInvariant(), optionPairParts[1].Trim(), lastKey); } else { MapToOption(options, optionPairParts[0].Trim().ToLowerInvariant(), optionPairParts[1].Trim()); lastKey = optionPairParts[0].Trim().ToLowerInvariant(); } } else { throw new ArgumentException( string.Format( "Unable to parse developer options which should be in the form of 'option=value&nextOption=nextValue'. The option '{0}' was not properly constructed", optionPair)); } } } return options; }
private bool ValidateDevCreds(EMRDeveloperOptions devOptions) { return !(string.IsNullOrWhiteSpace(devOptions.AccessKey) || string.IsNullOrWhiteSpace(devOptions.SecretAccessKey)); }
private OperationResult ParseDevOptions(string developerOptions, out EMRDeveloperOptions devOptions) { devOptions = null; var result = new OperationResult() { IsSuccess = false }; var progress = ""; try { progress += "Parsing developer options...\n"; devOptions = EMRDeveloperOptions.Parse(developerOptions); } catch (ArgumentException) { result.EndUserMessage = "Placeholder for ValidateDevCreds"; return result; } result.IsSuccess = true; result.EndUserMessage = progress; return result; }
private OperationResult EstablishClient(AddonManifest manifest, EMRDeveloperOptions devOptions, out IAmazonElasticMapReduce client) { OperationResult result; bool requireCreds; var accessKey = manifest.ProvisioningUsername; var secretAccessKey = manifest.ProvisioningPassword; var prop = manifest.Properties.First(p => p.Key.Equals("requireDevCredentials", StringComparison.InvariantCultureIgnoreCase)); //jobid = null; if (bool.TryParse(prop.Value, out requireCreds) && requireCreds) { if (!ValidateDevCreds(devOptions)) { client = null; result = new OperationResult() { IsSuccess = false, EndUserMessage = "The add on requires that developer credentials are specified but none were provided." }; return result; } accessKey = devOptions.AccessKey; secretAccessKey = devOptions.SecretAccessKey; } AWSCredentials credentials = new BasicAWSCredentials(accessKey, secretAccessKey); client = AWSClientFactory.CreateAmazonElasticMapReduceClient(credentials, RegionEndpoint.USEast1); //jobid = job.JobFlowId; result = new OperationResult() { IsSuccess = true }; return result; }
// Interior method takes in instance of DeveloperOptions (aptly named existingOptions) and maps them to the proper value. In essence, a setter method. public static void MapToOption(EMRDeveloperOptions existingOptions, string key, string value) { if ("accesskey".Equals(key)) { existingOptions.AccessKey = value; return; } if ("secretkey".Equals(key)) { existingOptions.SecretAccessKey = value; return; } if ("availabilityzone".Equals(key)) { existingOptions.AvailabilityZone = value; return; } if ("actiononfailure".Equals(key)) { existingOptions.ActionOnFailure = new Amazon.ElasticMapReduce.ActionOnFailure(value); return; } if ("ec2keyname".Equals(key)) { existingOptions.Ec2KeyName = value; return; } if ("enabledebugging".Equals(key)) { bool result; if (bool.TryParse(value, out result)) { throw new ArgumentException(string.Format("The developer option '{0}' can only have an integer value but '{1}' was used instead.", key, value)); } existingOptions.EnableDebugging = result; } if ("instancecount".Equals(key)) { int result; if (!int.TryParse(value, out result)) { throw new ArgumentException(string.Format("The developer option '{0}' can only have an integer value but '{1}' was used instead.", key, value)); } existingOptions.InstanceCount = result; return; } if ("jar".Equals(key)) { existingOptions.Jar = value; return; } if ("jobflowname".Equals(key)) { existingOptions.JobFlowName = value; return; } if ("args".Equals(key)) { existingOptions.Args.Add(value); return; } if ("loguri".Equals(key)) { existingOptions.LogURI = value; return; } if ("mainclass".Equals(key)) { existingOptions.MainClass = value; return; } if ("masterinstancetype".Equals(key)) { existingOptions.MasterInstanceType = value; return; } if ("slaveinstancetype".Equals(key)) { existingOptions.SlaveInstanceType = value; return; } if ("stepname".Equals(key)) { existingOptions.stepName = value; return; } if ("keepjobflowalivewhennosteps".Equals(key)) { bool result; if (!bool.TryParse(value, out result)) { throw new ArgumentException(string.Format("The developer option '{0}' must be a boolean value.", key)); } existingOptions.KeepJobFlowaliveWhenNoSteps = result; return; } if ("secretaccesskey".Equals(key)) { existingOptions.SecretAccessKey = value; return; } throw new ArgumentException(string.Format("The developer option '{0}' was not expected and is not understood.", key)); }