// Use this method to map all collections to their proper places.
// Usage here is to confirm that the subsequent key is the same as the preceding key.
// This forces that the REST call ensure all collection parameters are grouped together.
// Ex. This is good: (key1=value&key1=value2)
// Ex. This is bad: (key1=value&key2=value2)
// Ex. This is bad: (key1=value&key2=value2&key1=value3)
public static void MapToOptionWithCollection(EMRDeveloperOptions existingOptions, string key, string value, string lastKey)
{
if (key.Equals(lastKey))
{
if ("args".Equals(key))
{
existingOptions.Args.Add(value);
return;
}
throw new ArgumentException(string.Format("The developer option '{0}' was not expected and is not understood.", key));
}
throw new ArgumentException(string.Format("The developer option '{0}' is grouped out of order in the REST call. Group collection parameters together in the request.", key));
}
private OperationResult EstablishClient(AddonManifest manifest, EMRDeveloperOptions devOptions, out IAmazonElasticMapReduce client)
{
OperationResult result;
bool requireCreds;
var accessKey = manifest.ProvisioningUsername;
var secretAccessKey = manifest.ProvisioningPassword;
var prop = manifest.Properties.First(p => p.Key.Equals("requireDevCredentials", StringComparison.InvariantCultureIgnoreCase));
//jobid = null;
if (bool.TryParse(prop.Value, out requireCreds) && requireCreds)
{
if (!ValidateDevCreds(devOptions))
{
client = null;
result = new OperationResult()
{
IsSuccess = false,
EndUserMessage = "The add on requires that developer credentials are specified but none were provided."
};
return(result);
}
accessKey = devOptions.AccessKey;
secretAccessKey = devOptions.SecretAccessKey;
}
AWSCredentials credentials = new BasicAWSCredentials(accessKey, secretAccessKey);
client = AWSClientFactory.CreateAmazonElasticMapReduceClient(credentials, RegionEndpoint.USEast1);
//jobid = job.JobFlowId;
result = new OperationResult()
{
IsSuccess = true
};
return(result);
}
public override OperationResult Deprovision(AddonDeprovisionRequest request)
{
var deprovisionResult = new ProvisionAddOnResult("")
{
IsSuccess = true
};
deprovisionResult.ConnectionData = "deprovision";
AddonManifest manifest = request.Manifest;
string connectionData = request.ConnectionData;
string devOptions = request.DeveloperOptions;
//string jobid = null;
try
{
IAmazonElasticMapReduce client;
//var conInfo = ConnectionInfo.Parse(connectionData);
var developerOptions = EMRDeveloperOptions.Parse(devOptions);
var establishClientResult = EstablishClient(manifest, developerOptions, out client);
if (!establishClientResult.IsSuccess)
{
deprovisionResult.EndUserMessage = establishClientResult.EndUserMessage;
return(deprovisionResult);
}
var result = client.TerminateJobFlows(new TerminateJobFlowsRequest()
{
JobFlowIds = { connectionData }
});
deprovisionResult.IsSuccess = true;
deprovisionResult.EndUserMessage = "EMR Cluster Termination Request Successfully Invoked.";
}
catch (Exception)
{
deprovisionResult.EndUserMessage = "An error occurred during deprovisioning, please check the SOC logs for further assistance.";
}
return(deprovisionResult);
}
private OperationResult ParseDevOptions(string developerOptions, out EMRDeveloperOptions devOptions)
{
devOptions = null;
var result = new OperationResult()
{
IsSuccess = false
};
var progress = "";
try
{
progress += "Parsing developer options...\n";
devOptions = EMRDeveloperOptions.Parse(developerOptions);
}
catch (ArgumentException)
{
result.EndUserMessage = "Placeholder for ValidateDevCreds";
return(result);
}
result.IsSuccess = true;
result.EndUserMessage = progress;
return(result);
}
public static EMRDeveloperOptions Parse(string devOptions)
{
EMRDeveloperOptions options = new EMRDeveloperOptions();
String lastKey = "";
if (!string.IsNullOrWhiteSpace(devOptions))
{
// splitting all entries into arrays of optionPairs
var optionPairs = devOptions.Split(new[] { '&' }, StringSplitOptions.RemoveEmptyEntries);
foreach (var optionPair in optionPairs)
{
// splitting all optionPairs into arrays of key/value denominations
var optionPairParts = optionPair.Split(new[] { '=' }, StringSplitOptions.RemoveEmptyEntries);
if (optionPairParts.Length == 2)
{
if (lastKey.Equals(optionPairParts[0].Trim().ToLowerInvariant()))
{
MapToOptionWithCollection(options, optionPairParts[0].Trim().ToLowerInvariant(), optionPairParts[1].Trim(), lastKey);
}
else
{
MapToOption(options, optionPairParts[0].Trim().ToLowerInvariant(), optionPairParts[1].Trim());
lastKey = optionPairParts[0].Trim().ToLowerInvariant();
}
}
else
{
throw new ArgumentException(
string.Format(
"Unable to parse developer options which should be in the form of 'option=value&nextOption=nextValue'. The option '{0}' was not properly constructed",
optionPair));
}
}
}
return(options);
}
private bool ValidateDevCreds(EMRDeveloperOptions devOptions)
{
return(!(string.IsNullOrWhiteSpace(devOptions.AccessKey) || string.IsNullOrWhiteSpace(devOptions.SecretAccessKey)));
}
public override ProvisionAddOnResult Provision(AddonProvisionRequest request)
{
var provisionResult = new ProvisionAddOnResult("")
{
IsSuccess = true
};
AddonManifest manifest = request.Manifest;
string developerOptions = request.DeveloperOptions;
try
{
IAmazonElasticMapReduce client;
EMRDeveloperOptions devOptions;
var parseOptionsResult = ParseDevOptions(developerOptions, out devOptions);
if (!parseOptionsResult.IsSuccess)
{
provisionResult.EndUserMessage = parseOptionsResult.EndUserMessage;
return(provisionResult);
}
var establishClientResult = EstablishClient(manifest, EMRDeveloperOptions.Parse(developerOptions), out client);
if (!establishClientResult.IsSuccess)
{
provisionResult.EndUserMessage = establishClientResult.EndUserMessage;
return(provisionResult);
}
var stepFactory = new StepFactory();
StepConfig enabledebugging = null;
if (devOptions.EnableDebugging)
{
enabledebugging = new StepConfig
{
Name = "Enable debugging",
ActionOnFailure = "TERMINATE_JOB_FLOW",
HadoopJarStep = stepFactory.NewEnableDebuggingStep()
};
}
var installHive = new StepConfig
{
Name = "Install Hive",
ActionOnFailure = "TERMINATE_JOB_FLOW",
HadoopJarStep = stepFactory.NewInstallHiveStep()
};
var instanceConfig = new JobFlowInstancesConfig
{
Ec2KeyName = devOptions.Ec2KeyName,
HadoopVersion = "0.20",
InstanceCount = devOptions.InstanceCount,
// this is important. the EMR job flow must be kept alive for the application to see it during provisioning
KeepJobFlowAliveWhenNoSteps = true,
MasterInstanceType = devOptions.MasterInstanceType,
SlaveInstanceType = devOptions.SlaveInstanceType
};
var _request = new RunJobFlowRequest
{
Name = devOptions.JobFlowName,
Steps = { enabledebugging, installHive },
// revisit this one in ne
LogUri = "s3://myawsbucket",
Instances = instanceConfig
};
// if debugging is enabled, add to top of the list of steps.
if (devOptions.EnableDebugging)
{
_request.Steps.Insert(0, enabledebugging);
}
var result = client.RunJobFlow(_request);
// wait for JobFlowID to come back.
while (result.JobFlowId == null)
{
Thread.Sleep(1000);
}
provisionResult.IsSuccess = true;
provisionResult.ConnectionData = string.Format(result.JobFlowId);
}
catch (Exception e)
{
provisionResult.EndUserMessage = e.Message;
}
return(provisionResult);
}
// Interior method takes in instance of DeveloperOptions (aptly named existingOptions) and maps them to the proper value. In essence, a setter method.
public static void MapToOption(EMRDeveloperOptions existingOptions, string key, string value)
{
if ("accesskey".Equals(key))
{
existingOptions.AccessKey = value;
return;
}
if ("secretkey".Equals(key))
{
existingOptions.SecretAccessKey = value;
return;
}
if ("availabilityzone".Equals(key))
{
existingOptions.AvailabilityZone = value;
return;
}
if ("actiononfailure".Equals(key))
{
existingOptions.ActionOnFailure = new Amazon.ElasticMapReduce.ActionOnFailure(value);
return;
}
if ("ec2keyname".Equals(key))
{
existingOptions.Ec2KeyName = value;
return;
}
if("enabledebugging".Equals(key))
{
bool result;
if(bool.TryParse(value, out result))
{
throw new ArgumentException(string.Format("The developer option '{0}' can only have an integer value but '{1}' was used instead.", key, value));
}
existingOptions.EnableDebugging = result;
}
if ("instancecount".Equals(key))
{
int result;
if (!int.TryParse(value, out result))
{
throw new ArgumentException(string.Format("The developer option '{0}' can only have an integer value but '{1}' was used instead.", key, value));
}
existingOptions.InstanceCount = result;
return;
}
if ("jar".Equals(key))
{
existingOptions.Jar = value;
return;
}
if ("jobflowname".Equals(key))
{
existingOptions.JobFlowName = value;
return;
}
if ("args".Equals(key))
{
existingOptions.Args.Add(value);
return;
}
if ("loguri".Equals(key))
{
existingOptions.LogURI = value;
return;
}
if ("mainclass".Equals(key))
{
existingOptions.MainClass = value;
return;
}
if ("masterinstancetype".Equals(key))
{
existingOptions.MasterInstanceType = value;
return;
}
if ("slaveinstancetype".Equals(key))
{
existingOptions.SlaveInstanceType = value;
return;
}
if ("stepname".Equals(key))
{
existingOptions.stepName = value;
return;
}
if ("keepjobflowalivewhennosteps".Equals(key))
{
bool result;
if (!bool.TryParse(value, out result))
{
throw new ArgumentException(string.Format("The developer option '{0}' must be a boolean value.", key));
}
existingOptions.KeepJobFlowaliveWhenNoSteps = result;
return;
}
if ("secretaccesskey".Equals(key))
{
existingOptions.SecretAccessKey = value;
return;
}
throw new ArgumentException(string.Format("The developer option '{0}' was not expected and is not understood.", key));
}
public static EMRDeveloperOptions Parse(string devOptions)
{
EMRDeveloperOptions options = new EMRDeveloperOptions();
String lastKey = "";
if (!string.IsNullOrWhiteSpace(devOptions))
{
// splitting all entries into arrays of optionPairs
var optionPairs = devOptions.Split(new[] { '&' }, StringSplitOptions.RemoveEmptyEntries);
foreach (var optionPair in optionPairs)
{
// splitting all optionPairs into arrays of key/value denominations
var optionPairParts = optionPair.Split(new[] { '=' }, StringSplitOptions.RemoveEmptyEntries);
if (optionPairParts.Length == 2)
{
if (lastKey.Equals(optionPairParts[0].Trim().ToLowerInvariant()))
{
MapToOptionWithCollection(options, optionPairParts[0].Trim().ToLowerInvariant(), optionPairParts[1].Trim(), lastKey);
}
else
{
MapToOption(options, optionPairParts[0].Trim().ToLowerInvariant(), optionPairParts[1].Trim());
lastKey = optionPairParts[0].Trim().ToLowerInvariant();
}
}
else
{
throw new ArgumentException(
string.Format(
"Unable to parse developer options which should be in the form of 'option=value&nextOption=nextValue'. The option '{0}' was not properly constructed",
optionPair));
}
}
}
return options;
}
// Use this method to map all collections to their proper places.
// Usage here is to confirm that the subsequent key is the same as the preceding key.
// This forces that the REST call ensure all collection parameters are grouped together.
// Ex. This is good: (key1=value&key1=value2)
// Ex. This is bad: (key1=value&key2=value2)
// Ex. This is bad: (key1=value&key2=value2&key1=value3)
public static void MapToOptionWithCollection(EMRDeveloperOptions existingOptions, string key, string value, string lastKey)
{
if (key.Equals(lastKey))
{
if ("args".Equals(key))
{
existingOptions.Args.Add(value);
return;
}
throw new ArgumentException(string.Format("The developer option '{0}' was not expected and is not understood.", key));
}
throw new ArgumentException(string.Format("The developer option '{0}' is grouped out of order in the REST call. Group collection parameters together in the request.", key));
}
private bool ValidateDevCreds(EMRDeveloperOptions devOptions)
{
return !(string.IsNullOrWhiteSpace(devOptions.AccessKey) || string.IsNullOrWhiteSpace(devOptions.SecretAccessKey));
}
private OperationResult ParseDevOptions(string developerOptions, out EMRDeveloperOptions devOptions)
{
devOptions = null;
var result = new OperationResult() { IsSuccess = false };
var progress = "";
try
{
progress += "Parsing developer options...\n";
devOptions = EMRDeveloperOptions.Parse(developerOptions);
}
catch (ArgumentException)
{
result.EndUserMessage = "Placeholder for ValidateDevCreds";
return result;
}
result.IsSuccess = true;
result.EndUserMessage = progress;
return result;
}
private OperationResult EstablishClient(AddonManifest manifest, EMRDeveloperOptions devOptions, out IAmazonElasticMapReduce client)
{
OperationResult result;
bool requireCreds;
var accessKey = manifest.ProvisioningUsername;
var secretAccessKey = manifest.ProvisioningPassword;
var prop = manifest.Properties.First(p => p.Key.Equals("requireDevCredentials", StringComparison.InvariantCultureIgnoreCase));
//jobid = null;
if (bool.TryParse(prop.Value, out requireCreds) && requireCreds)
{
if (!ValidateDevCreds(devOptions))
{
client = null;
result = new OperationResult()
{
IsSuccess = false,
EndUserMessage = "The add on requires that developer credentials are specified but none were provided."
};
return result;
}
accessKey = devOptions.AccessKey;
secretAccessKey = devOptions.SecretAccessKey;
}
AWSCredentials credentials = new BasicAWSCredentials(accessKey, secretAccessKey);
client = AWSClientFactory.CreateAmazonElasticMapReduceClient(credentials, RegionEndpoint.USEast1);
//jobid = job.JobFlowId;
result = new OperationResult()
{
IsSuccess = true
};
return result;
}
// Interior method takes in instance of DeveloperOptions (aptly named existingOptions) and maps them to the proper value. In essence, a setter method.
public static void MapToOption(EMRDeveloperOptions existingOptions, string key, string value)
{
if ("accesskey".Equals(key))
{
existingOptions.AccessKey = value;
return;
}
if ("secretkey".Equals(key))
{
existingOptions.SecretAccessKey = value;
return;
}
if ("availabilityzone".Equals(key))
{
existingOptions.AvailabilityZone = value;
return;
}
if ("actiononfailure".Equals(key))
{
existingOptions.ActionOnFailure = new Amazon.ElasticMapReduce.ActionOnFailure(value);
return;
}
if ("ec2keyname".Equals(key))
{
existingOptions.Ec2KeyName = value;
return;
}
if ("enabledebugging".Equals(key))
{
bool result;
if (bool.TryParse(value, out result))
{
throw new ArgumentException(string.Format("The developer option '{0}' can only have an integer value but '{1}' was used instead.", key, value));
}
existingOptions.EnableDebugging = result;
}
if ("instancecount".Equals(key))
{
int result;
if (!int.TryParse(value, out result))
{
throw new ArgumentException(string.Format("The developer option '{0}' can only have an integer value but '{1}' was used instead.", key, value));
}
existingOptions.InstanceCount = result;
return;
}
if ("jar".Equals(key))
{
existingOptions.Jar = value;
return;
}
if ("jobflowname".Equals(key))
{
existingOptions.JobFlowName = value;
return;
}
if ("args".Equals(key))
{
existingOptions.Args.Add(value);
return;
}
if ("loguri".Equals(key))
{
existingOptions.LogURI = value;
return;
}
if ("mainclass".Equals(key))
{
existingOptions.MainClass = value;
return;
}
if ("masterinstancetype".Equals(key))
{
existingOptions.MasterInstanceType = value;
return;
}
if ("slaveinstancetype".Equals(key))
{
existingOptions.SlaveInstanceType = value;
return;
}
if ("stepname".Equals(key))
{
existingOptions.stepName = value;
return;
}
if ("keepjobflowalivewhennosteps".Equals(key))
{
bool result;
if (!bool.TryParse(value, out result))
{
throw new ArgumentException(string.Format("The developer option '{0}' must be a boolean value.", key));
}
existingOptions.KeepJobFlowaliveWhenNoSteps = result;
return;
}
if ("secretaccesskey".Equals(key))
{
existingOptions.SecretAccessKey = value;
return;
}
throw new ArgumentException(string.Format("The developer option '{0}' was not expected and is not understood.", key));
}