public void VisitJobFlow()
{
//Init args
JobFlow jobFlow = new JobFlow();
jobFlow.Name = "testName-{jobFlowId}";
jobFlow.LogUri = "{myBucket}/logs/";
jobFlow.JobFlowRole = "arn:{myRole}";
jobFlow.AmiVersion = "{amiVersion}";
jobFlow.AdditionalInfo = "{ name: \"name1\", contact: \"{contact}\" }";
jobFlow.Ec2KeyName = "{ec2Key}";
jobFlow.HadoopVersion = "{hadoopVersion}";
jobFlow.KeepJobFlowAliveWhenNoSteps = true;
jobFlow.TerminationProtected = true;
jobFlow.MasterInstanceType = "{masterInstanceType}";
jobFlow.SlaveInstanceType = "{slaveInstanceType}";
jobFlow.InstanceCount = 34;
//Init visitor
BuildRequestVisitor visitor = new BuildRequestVisitor(BuildRequestVisitorTest.GetSettings());
VisitorSubscriber visitorSubscriber = new VisitorSubscriber(visitor);
//Action
jobFlow.Accept(visitor);
//Verify
Assert.AreEqual(2, visitorSubscriber.TotalObjCount, "Unexpected number of objects created");
RunJobFlowRequest actualJobFlowRequest = visitorSubscriber.jobFlowRequestList[0];
Assert.AreEqual("testName-j-111AAABBBNJ2I", actualJobFlowRequest.Name, "Unexpected Name");
Assert.AreEqual("s3://myBucket/logs/", actualJobFlowRequest.LogUri, "Unexpected LogUri");
Assert.AreEqual("arn:SupperSlonic", actualJobFlowRequest.JobFlowRole, "Unexpected JobFlowRole");
Assert.AreEqual("3.0.3", actualJobFlowRequest.AmiVersion, "Unexpected AmiVersion");
Assert.AreEqual("{ name: \"name1\", contact: \"supperslonic.com\" }", actualJobFlowRequest.AdditionalInfo, "Unexpected AdditionalInfo");
JobFlowInstancesConfig actualJobFlowInstancesConfig = visitorSubscriber.instanceConfigList[0];
Assert.AreEqual("testEC2Key", actualJobFlowInstancesConfig.Ec2KeyName, "Unexpected Ec2KeyName");
Assert.AreEqual("2.2.0", actualJobFlowInstancesConfig.HadoopVersion, "Unexpected HadoopVersion");
Assert.IsTrue(actualJobFlowInstancesConfig.KeepJobFlowAliveWhenNoSteps, "Unexpected KeepJobFlowAliveWhenNoSteps");
Assert.IsTrue(actualJobFlowInstancesConfig.TerminationProtected, "Unexpected TerminationProtected");
Assert.AreEqual("m1.medium", actualJobFlowInstancesConfig.MasterInstanceType, "Unexpected MasterInstanceType");
Assert.AreEqual("m3.2xlarge", actualJobFlowInstancesConfig.SlaveInstanceType, "Unexpected SlaveInstanceType");
Assert.AreEqual(34, actualJobFlowInstancesConfig.InstanceCount, "Unexpected InstanceCount");
}
/**
* Convert RunJobFlowRequest to name value pairs
*/
private static IDictionary <string, string> ConvertRunJobFlow(RunJobFlowRequest request)
{
IDictionary <string, string> parameters = new Dictionary <string, string>();
parameters["Action"] = "RunJobFlow";
if (request.IsSetName())
{
parameters["Name"] = request.Name;
}
if (request.IsSetLogUri())
{
parameters["LogUri"] = request.LogUri;
}
if (request.IsSetAdditionalInfo())
{
parameters["AdditionalInfo"] = request.AdditionalInfo;
}
if (request.IsSetInstances())
{
JobFlowInstancesConfig runJobFlowRequestInstances = request.Instances;
if (runJobFlowRequestInstances.IsSetMasterInstanceType())
{
parameters[String.Concat("Instances", ".", "MasterInstanceType")] = runJobFlowRequestInstances.MasterInstanceType;
}
if (runJobFlowRequestInstances.IsSetSlaveInstanceType())
{
parameters[String.Concat("Instances", ".", "SlaveInstanceType")] = runJobFlowRequestInstances.SlaveInstanceType;
}
if (runJobFlowRequestInstances.IsSetInstanceCount())
{
parameters[String.Concat("Instances", ".", "InstanceCount")] = runJobFlowRequestInstances.InstanceCount.ToString();
}
if (runJobFlowRequestInstances.IsSetEc2KeyName())
{
parameters[String.Concat("Instances", ".", "Ec2KeyName")] = runJobFlowRequestInstances.Ec2KeyName;
}
if (runJobFlowRequestInstances.IsSetPlacement())
{
PlacementType instancesPlacement = runJobFlowRequestInstances.Placement;
if (instancesPlacement.IsSetAvailabilityZone())
{
parameters[String.Concat("Instances", ".", "Placement", ".", "AvailabilityZone")] = instancesPlacement.AvailabilityZone;
}
}
if (runJobFlowRequestInstances.IsSetKeepJobFlowAliveWhenNoSteps())
{
parameters[String.Concat("Instances", ".", "KeepJobFlowAliveWhenNoSteps")] = runJobFlowRequestInstances.KeepJobFlowAliveWhenNoSteps.ToString().ToLower();
}
if (runJobFlowRequestInstances.IsSetHadoopVersion())
{
parameters[String.Concat("Instances", ".", "HadoopVersion")] = runJobFlowRequestInstances.HadoopVersion;
}
}
List <StepConfig> runJobFlowRequestStepsList = request.Steps;
int runJobFlowRequestStepsListIndex = 1;
foreach (StepConfig runJobFlowRequestSteps in runJobFlowRequestStepsList)
{
if (runJobFlowRequestSteps.IsSetName())
{
parameters[String.Concat("Steps", ".member.", runJobFlowRequestStepsListIndex, ".", "Name")] = runJobFlowRequestSteps.Name;
}
if (runJobFlowRequestSteps.IsSetActionOnFailure())
{
parameters[String.Concat("Steps", ".member.", runJobFlowRequestStepsListIndex, ".", "ActionOnFailure")] = runJobFlowRequestSteps.ActionOnFailure;
}
if (runJobFlowRequestSteps.IsSetHadoopJarStep())
{
HadoopJarStepConfig stepsHadoopJarStep = runJobFlowRequestSteps.HadoopJarStep;
List <KeyValue> hadoopJarStepPropertiesList = stepsHadoopJarStep.Properties;
int hadoopJarStepPropertiesListIndex = 1;
foreach (KeyValue hadoopJarStepProperties in hadoopJarStepPropertiesList)
{
if (hadoopJarStepProperties.IsSetKey())
{
parameters[String.Concat("Steps", ".member.", runJobFlowRequestStepsListIndex, ".", "HadoopJarStep", ".", "Properties", ".member.", hadoopJarStepPropertiesListIndex, ".", "Key")] = hadoopJarStepProperties.Key;
}
if (hadoopJarStepProperties.IsSetValue())
{
parameters[String.Concat("Steps", ".member.", runJobFlowRequestStepsListIndex, ".", "HadoopJarStep", ".", "Properties", ".member.", hadoopJarStepPropertiesListIndex, ".", "Value")] = hadoopJarStepProperties.Value;
}
hadoopJarStepPropertiesListIndex++;
}
if (stepsHadoopJarStep.IsSetJar())
{
parameters[String.Concat("Steps", ".member.", runJobFlowRequestStepsListIndex, ".", "HadoopJarStep", ".", "Jar")] = stepsHadoopJarStep.Jar;
}
if (stepsHadoopJarStep.IsSetMainClass())
{
parameters[String.Concat("Steps", ".member.", runJobFlowRequestStepsListIndex, ".", "HadoopJarStep", ".", "MainClass")] = stepsHadoopJarStep.MainClass;
}
List <string> hadoopJarStepArgsList = stepsHadoopJarStep.Args;
int hadoopJarStepArgsListIndex = 1;
foreach (string hadoopJarStepArgs in hadoopJarStepArgsList)
{
parameters[String.Concat("Steps", ".member.", runJobFlowRequestStepsListIndex, ".", "HadoopJarStep", ".", "Args", ".member.", hadoopJarStepArgsListIndex)] = hadoopJarStepArgs;
hadoopJarStepArgsListIndex++;
}
}
runJobFlowRequestStepsListIndex++;
}
List <BootstrapActionConfig> runJobFlowRequestBootstrapActionsList = request.BootstrapActions;
int runJobFlowRequestBootstrapActionsListIndex = 1;
foreach (BootstrapActionConfig runJobFlowRequestBootstrapActions in runJobFlowRequestBootstrapActionsList)
{
if (runJobFlowRequestBootstrapActions.IsSetName())
{
parameters[String.Concat("BootstrapActions", ".member.", runJobFlowRequestBootstrapActionsListIndex, ".", "Name")] = runJobFlowRequestBootstrapActions.Name;
}
if (runJobFlowRequestBootstrapActions.IsSetScriptBootstrapAction())
{
ScriptBootstrapActionConfig bootstrapActionsScriptBootstrapAction = runJobFlowRequestBootstrapActions.ScriptBootstrapAction;
if (bootstrapActionsScriptBootstrapAction.IsSetPath())
{
parameters[String.Concat("BootstrapActions", ".member.", runJobFlowRequestBootstrapActionsListIndex, ".", "ScriptBootstrapAction", ".", "Path")] = bootstrapActionsScriptBootstrapAction.Path;
}
List <string> scriptBootstrapActionArgsList = bootstrapActionsScriptBootstrapAction.Args;
int scriptBootstrapActionArgsListIndex = 1;
foreach (string scriptBootstrapActionArgs in scriptBootstrapActionArgsList)
{
parameters[String.Concat("BootstrapActions", ".member.", runJobFlowRequestBootstrapActionsListIndex, ".", "ScriptBootstrapAction", ".", "Args", ".member.", scriptBootstrapActionArgsListIndex)] = scriptBootstrapActionArgs;
scriptBootstrapActionArgsListIndex++;
}
}
runJobFlowRequestBootstrapActionsListIndex++;
}
return(parameters);
}
public override ProvisionAddOnResult Provision(AddonProvisionRequest request)
{
var provisionResult = new ProvisionAddOnResult("")
{
IsSuccess = true
};
AddonManifest manifest = request.Manifest;
string developerOptions = request.DeveloperOptions;
try
{
IAmazonElasticMapReduce client;
EMRDeveloperOptions devOptions;
var parseOptionsResult = ParseDevOptions(developerOptions, out devOptions);
if (!parseOptionsResult.IsSuccess)
{
provisionResult.EndUserMessage = parseOptionsResult.EndUserMessage;
return(provisionResult);
}
var establishClientResult = EstablishClient(manifest, EMRDeveloperOptions.Parse(developerOptions), out client);
if (!establishClientResult.IsSuccess)
{
provisionResult.EndUserMessage = establishClientResult.EndUserMessage;
return(provisionResult);
}
var stepFactory = new StepFactory();
StepConfig enabledebugging = null;
if (devOptions.EnableDebugging)
{
enabledebugging = new StepConfig
{
Name = "Enable debugging",
ActionOnFailure = "TERMINATE_JOB_FLOW",
HadoopJarStep = stepFactory.NewEnableDebuggingStep()
};
}
var installHive = new StepConfig
{
Name = "Install Hive",
ActionOnFailure = "TERMINATE_JOB_FLOW",
HadoopJarStep = stepFactory.NewInstallHiveStep()
};
var instanceConfig = new JobFlowInstancesConfig
{
Ec2KeyName = devOptions.Ec2KeyName,
HadoopVersion = "0.20",
InstanceCount = devOptions.InstanceCount,
// this is important. the EMR job flow must be kept alive for the application to see it during provisioning
KeepJobFlowAliveWhenNoSteps = true,
MasterInstanceType = devOptions.MasterInstanceType,
SlaveInstanceType = devOptions.SlaveInstanceType
};
var _request = new RunJobFlowRequest
{
Name = devOptions.JobFlowName,
Steps = { enabledebugging, installHive },
// revisit this one in ne
LogUri = "s3://myawsbucket",
Instances = instanceConfig
};
// if debugging is enabled, add to top of the list of steps.
if (devOptions.EnableDebugging)
{
_request.Steps.Insert(0, enabledebugging);
}
var result = client.RunJobFlow(_request);
// wait for JobFlowID to come back.
while (result.JobFlowId == null)
{
Thread.Sleep(1000);
}
provisionResult.IsSuccess = true;
provisionResult.ConnectionData = string.Format(result.JobFlowId);
}
catch (Exception e)
{
provisionResult.EndUserMessage = e.Message;
}
return(provisionResult);
}
private static IDictionary <string, string> ConvertRunJobFlow(RunJobFlowRequest request)
{
IDictionary <string, string> dictionary = new Dictionary <string, string>();
dictionary["Action"] = "RunJobFlow";
if (request.IsSetName())
{
dictionary["Name"] = request.Name;
}
if (request.IsSetLogUri())
{
dictionary["LogUri"] = request.LogUri;
}
if (request.IsSetAdditionalInfo())
{
dictionary["AdditionalInfo"] = request.AdditionalInfo;
}
if (request.IsSetInstances())
{
JobFlowInstancesConfig instances = request.Instances;
if (instances.IsSetMasterInstanceType())
{
dictionary["Instances" + "." + "MasterInstanceType"] = instances.MasterInstanceType;
}
if (instances.IsSetSlaveInstanceType())
{
dictionary["Instances" + "." + "SlaveInstanceType"] = instances.SlaveInstanceType;
}
if (instances.IsSetInstanceCount())
{
dictionary["Instances" + "." + "InstanceCount"] = instances.InstanceCount.ToString();
}
if (instances.IsSetEc2KeyName())
{
dictionary["Instances" + "." + "Ec2KeyName"] = instances.Ec2KeyName;
}
if (instances.IsSetPlacement())
{
PlacementType placement = instances.Placement;
if (placement.IsSetAvailabilityZone())
{
dictionary["Instances" + "." + "Placement" + "." + "AvailabilityZone"] = placement.AvailabilityZone;
}
}
if (instances.IsSetKeepJobFlowAliveWhenNoSteps())
{
dictionary["Instances" + "." + "KeepJobFlowAliveWhenNoSteps"] = instances.KeepJobFlowAliveWhenNoSteps.ToString().ToLower();
}
if (instances.IsSetHadoopVersion())
{
dictionary["Instances" + "." + "HadoopVersion"] = instances.HadoopVersion;
}
}
List <StepConfig> steps = request.Steps;
int num = 1;
foreach (StepConfig config2 in steps)
{
if (config2.IsSetName())
{
dictionary[string.Concat(new object[] { "Steps", ".member.", num, ".", "Name" })] = config2.Name;
}
if (config2.IsSetActionOnFailure())
{
dictionary[string.Concat(new object[] { "Steps", ".member.", num, ".", "ActionOnFailure" })] = config2.ActionOnFailure;
}
if (config2.IsSetHadoopJarStep())
{
HadoopJarStepConfig hadoopJarStep = config2.HadoopJarStep;
List <KeyValue> properties = hadoopJarStep.Properties;
int num2 = 1;
foreach (KeyValue value2 in properties)
{
if (value2.IsSetKey())
{
dictionary[string.Concat(new object[] { "Steps", ".member.", num, ".", "HadoopJarStep", ".", "Properties", ".member.", num2, ".", "Key" })] = value2.Key;
}
if (value2.IsSetValue())
{
dictionary[string.Concat(new object[] { "Steps", ".member.", num, ".", "HadoopJarStep", ".", "Properties", ".member.", num2, ".", "Value" })] = value2.Value;
}
num2++;
}
if (hadoopJarStep.IsSetJar())
{
dictionary[string.Concat(new object[] { "Steps", ".member.", num, ".", "HadoopJarStep", ".", "Jar" })] = hadoopJarStep.Jar;
}
if (hadoopJarStep.IsSetMainClass())
{
dictionary[string.Concat(new object[] { "Steps", ".member.", num, ".", "HadoopJarStep", ".", "MainClass" })] = hadoopJarStep.MainClass;
}
List <string> args = hadoopJarStep.Args;
int num3 = 1;
foreach (string str in args)
{
dictionary[string.Concat(new object[] { "Steps", ".member.", num, ".", "HadoopJarStep", ".", "Args", ".member.", num3 })] = str;
num3++;
}
}
num++;
}
List <BootstrapActionConfig> bootstrapActions = request.BootstrapActions;
int num4 = 1;
foreach (BootstrapActionConfig config4 in bootstrapActions)
{
if (config4.IsSetName())
{
dictionary[string.Concat(new object[] { "BootstrapActions", ".member.", num4, ".", "Name" })] = config4.Name;
}
if (config4.IsSetScriptBootstrapAction())
{
ScriptBootstrapActionConfig scriptBootstrapAction = config4.ScriptBootstrapAction;
if (scriptBootstrapAction.IsSetPath())
{
dictionary[string.Concat(new object[] { "BootstrapActions", ".member.", num4, ".", "ScriptBootstrapAction", ".", "Path" })] = scriptBootstrapAction.Path;
}
List <string> list5 = scriptBootstrapAction.Args;
int num5 = 1;
foreach (string str2 in list5)
{
dictionary[string.Concat(new object[] { "BootstrapActions", ".member.", num4, ".", "ScriptBootstrapAction", ".", "Args", ".member.", num5 })] = str2;
num5++;
}
}
num4++;
}
return(dictionary);
}
public IRequest Marshall(RunJobFlowRequest runJobFlowRequest)
{
IRequest request = new DefaultRequest(runJobFlowRequest, "AmazonElasticMapReduce");
string target = "ElasticMapReduce.RunJobFlow";
request.Headers["X-Amz-Target"] = target;
request.Headers["Content-Type"] = "application/x-amz-json-1.1";
string uriResourcePath = "";
request.ResourcePath = uriResourcePath;
using (StringWriter stringWriter = new StringWriter(CultureInfo.InvariantCulture))
{
JsonWriter writer = new JsonWriter(stringWriter);
writer.WriteObjectStart();
if (runJobFlowRequest != null && runJobFlowRequest.IsSetName())
{
writer.WritePropertyName("Name");
writer.Write(runJobFlowRequest.Name);
}
if (runJobFlowRequest != null && runJobFlowRequest.IsSetLogUri())
{
writer.WritePropertyName("LogUri");
writer.Write(runJobFlowRequest.LogUri);
}
if (runJobFlowRequest != null && runJobFlowRequest.IsSetAdditionalInfo())
{
writer.WritePropertyName("AdditionalInfo");
writer.Write(runJobFlowRequest.AdditionalInfo);
}
if (runJobFlowRequest != null && runJobFlowRequest.IsSetAmiVersion())
{
writer.WritePropertyName("AmiVersion");
writer.Write(runJobFlowRequest.AmiVersion);
}
if (runJobFlowRequest != null)
{
JobFlowInstancesConfig instances = runJobFlowRequest.Instances;
if (instances != null)
{
writer.WritePropertyName("Instances");
writer.WriteObjectStart();
if (instances != null && instances.IsSetMasterInstanceType())
{
writer.WritePropertyName("MasterInstanceType");
writer.Write(instances.MasterInstanceType);
}
if (instances != null && instances.IsSetSlaveInstanceType())
{
writer.WritePropertyName("SlaveInstanceType");
writer.Write(instances.SlaveInstanceType);
}
if (instances != null && instances.IsSetInstanceCount())
{
writer.WritePropertyName("InstanceCount");
writer.Write(instances.InstanceCount);
}
if (instances != null && instances.InstanceGroups != null && instances.InstanceGroups.Count > 0)
{
List <InstanceGroupConfig> instanceGroupsList = instances.InstanceGroups;
writer.WritePropertyName("InstanceGroups");
writer.WriteArrayStart();
foreach (InstanceGroupConfig instanceGroupsListValue in instanceGroupsList)
{
writer.WriteObjectStart();
if (instanceGroupsListValue != null && instanceGroupsListValue.IsSetName())
{
writer.WritePropertyName("Name");
writer.Write(instanceGroupsListValue.Name);
}
if (instanceGroupsListValue != null && instanceGroupsListValue.IsSetMarket())
{
writer.WritePropertyName("Market");
writer.Write(instanceGroupsListValue.Market);
}
if (instanceGroupsListValue != null && instanceGroupsListValue.IsSetInstanceRole())
{
writer.WritePropertyName("InstanceRole");
writer.Write(instanceGroupsListValue.InstanceRole);
}
if (instanceGroupsListValue != null && instanceGroupsListValue.IsSetBidPrice())
{
writer.WritePropertyName("BidPrice");
writer.Write(instanceGroupsListValue.BidPrice);
}
if (instanceGroupsListValue != null && instanceGroupsListValue.IsSetInstanceType())
{
writer.WritePropertyName("InstanceType");
writer.Write(instanceGroupsListValue.InstanceType);
}
if (instanceGroupsListValue != null && instanceGroupsListValue.IsSetInstanceCount())
{
writer.WritePropertyName("InstanceCount");
writer.Write(instanceGroupsListValue.InstanceCount);
}
writer.WriteObjectEnd();
}
writer.WriteArrayEnd();
}
if (instances != null && instances.IsSetEc2KeyName())
{
writer.WritePropertyName("Ec2KeyName");
writer.Write(instances.Ec2KeyName);
}
if (instances != null)
{
PlacementType placement = instances.Placement;
if (placement != null)
{
writer.WritePropertyName("Placement");
writer.WriteObjectStart();
if (placement != null && placement.IsSetAvailabilityZone())
{
writer.WritePropertyName("AvailabilityZone");
writer.Write(placement.AvailabilityZone);
}
writer.WriteObjectEnd();
}
}
if (instances != null && instances.IsSetKeepJobFlowAliveWhenNoSteps())
{
writer.WritePropertyName("KeepJobFlowAliveWhenNoSteps");
writer.Write(instances.KeepJobFlowAliveWhenNoSteps);
}
if (instances != null && instances.IsSetTerminationProtected())
{
writer.WritePropertyName("TerminationProtected");
writer.Write(instances.TerminationProtected);
}
if (instances != null && instances.IsSetHadoopVersion())
{
writer.WritePropertyName("HadoopVersion");
writer.Write(instances.HadoopVersion);
}
if (instances != null && instances.IsSetEc2SubnetId())
{
writer.WritePropertyName("Ec2SubnetId");
writer.Write(instances.Ec2SubnetId);
}
writer.WriteObjectEnd();
}
}
if (runJobFlowRequest != null && runJobFlowRequest.Steps != null && runJobFlowRequest.Steps.Count > 0)
{
List <StepConfig> stepsList = runJobFlowRequest.Steps;
writer.WritePropertyName("Steps");
writer.WriteArrayStart();
foreach (StepConfig stepsListValue in stepsList)
{
writer.WriteObjectStart();
if (stepsListValue != null && stepsListValue.IsSetName())
{
writer.WritePropertyName("Name");
writer.Write(stepsListValue.Name);
}
if (stepsListValue != null && stepsListValue.IsSetActionOnFailure())
{
writer.WritePropertyName("ActionOnFailure");
writer.Write(stepsListValue.ActionOnFailure);
}
if (stepsListValue != null)
{
HadoopJarStepConfig hadoopJarStep = stepsListValue.HadoopJarStep;
if (hadoopJarStep != null)
{
writer.WritePropertyName("HadoopJarStep");
writer.WriteObjectStart();
if (hadoopJarStep != null && hadoopJarStep.Properties != null && hadoopJarStep.Properties.Count > 0)
{
List <KeyValue> propertiesList = hadoopJarStep.Properties;
writer.WritePropertyName("Properties");
writer.WriteArrayStart();
foreach (KeyValue propertiesListValue in propertiesList)
{
writer.WriteObjectStart();
if (propertiesListValue != null && propertiesListValue.IsSetKey())
{
writer.WritePropertyName("Key");
writer.Write(propertiesListValue.Key);
}
if (propertiesListValue != null && propertiesListValue.IsSetValue())
{
writer.WritePropertyName("Value");
writer.Write(propertiesListValue.Value);
}
writer.WriteObjectEnd();
}
writer.WriteArrayEnd();
}
if (hadoopJarStep != null && hadoopJarStep.IsSetJar())
{
writer.WritePropertyName("Jar");
writer.Write(hadoopJarStep.Jar);
}
if (hadoopJarStep != null && hadoopJarStep.IsSetMainClass())
{
writer.WritePropertyName("MainClass");
writer.Write(hadoopJarStep.MainClass);
}
if (hadoopJarStep != null && hadoopJarStep.Args != null && hadoopJarStep.Args.Count > 0)
{
List <string> argsList = hadoopJarStep.Args;
writer.WritePropertyName("Args");
writer.WriteArrayStart();
foreach (string argsListValue in argsList)
{
writer.Write(StringUtils.FromString(argsListValue));
}
writer.WriteArrayEnd();
}
writer.WriteObjectEnd();
}
}
writer.WriteObjectEnd();
}
writer.WriteArrayEnd();
}
if (runJobFlowRequest != null && runJobFlowRequest.BootstrapActions != null && runJobFlowRequest.BootstrapActions.Count > 0)
{
List <BootstrapActionConfig> bootstrapActionsList = runJobFlowRequest.BootstrapActions;
writer.WritePropertyName("BootstrapActions");
writer.WriteArrayStart();
foreach (BootstrapActionConfig bootstrapActionsListValue in bootstrapActionsList)
{
writer.WriteObjectStart();
if (bootstrapActionsListValue != null && bootstrapActionsListValue.IsSetName())
{
writer.WritePropertyName("Name");
writer.Write(bootstrapActionsListValue.Name);
}
if (bootstrapActionsListValue != null)
{
ScriptBootstrapActionConfig scriptBootstrapAction = bootstrapActionsListValue.ScriptBootstrapAction;
if (scriptBootstrapAction != null)
{
writer.WritePropertyName("ScriptBootstrapAction");
writer.WriteObjectStart();
if (scriptBootstrapAction != null && scriptBootstrapAction.IsSetPath())
{
writer.WritePropertyName("Path");
writer.Write(scriptBootstrapAction.Path);
}
if (scriptBootstrapAction != null && scriptBootstrapAction.Args != null && scriptBootstrapAction.Args.Count > 0)
{
List <string> argsList = scriptBootstrapAction.Args;
writer.WritePropertyName("Args");
writer.WriteArrayStart();
foreach (string argsListValue in argsList)
{
writer.Write(StringUtils.FromString(argsListValue));
}
writer.WriteArrayEnd();
}
writer.WriteObjectEnd();
}
}
writer.WriteObjectEnd();
}
writer.WriteArrayEnd();
}
if (runJobFlowRequest != null && runJobFlowRequest.SupportedProducts != null && runJobFlowRequest.SupportedProducts.Count > 0)
{
List <string> supportedProductsList = runJobFlowRequest.SupportedProducts;
writer.WritePropertyName("SupportedProducts");
writer.WriteArrayStart();
foreach (string supportedProductsListValue in supportedProductsList)
{
writer.Write(StringUtils.FromString(supportedProductsListValue));
}
writer.WriteArrayEnd();
}
if (runJobFlowRequest != null && runJobFlowRequest.NewSupportedProducts != null && runJobFlowRequest.NewSupportedProducts.Count > 0)
{
List <SupportedProductConfig> newSupportedProductsList = runJobFlowRequest.NewSupportedProducts;
writer.WritePropertyName("NewSupportedProducts");
writer.WriteArrayStart();
foreach (SupportedProductConfig newSupportedProductsListValue in newSupportedProductsList)
{
writer.WriteObjectStart();
if (newSupportedProductsListValue != null && newSupportedProductsListValue.IsSetName())
{
writer.WritePropertyName("Name");
writer.Write(newSupportedProductsListValue.Name);
}
if (newSupportedProductsListValue != null && newSupportedProductsListValue.Args != null && newSupportedProductsListValue.Args.Count > 0)
{
List <string> argsList = newSupportedProductsListValue.Args;
writer.WritePropertyName("Args");
writer.WriteArrayStart();
foreach (string argsListValue in argsList)
{
writer.Write(StringUtils.FromString(argsListValue));
}
writer.WriteArrayEnd();
}
writer.WriteObjectEnd();
}
writer.WriteArrayEnd();
}
if (runJobFlowRequest != null && runJobFlowRequest.IsSetVisibleToAllUsers())
{
writer.WritePropertyName("VisibleToAllUsers");
writer.Write(runJobFlowRequest.VisibleToAllUsers);
}
if (runJobFlowRequest != null && runJobFlowRequest.IsSetJobFlowRole())
{
writer.WritePropertyName("JobFlowRole");
writer.Write(runJobFlowRequest.JobFlowRole);
}
if (runJobFlowRequest != null && runJobFlowRequest.Tags != null && runJobFlowRequest.Tags.Count > 0)
{
List <Tag> tagsList = runJobFlowRequest.Tags;
writer.WritePropertyName("Tags");
writer.WriteArrayStart();
foreach (Tag tagsListValue in tagsList)
{
writer.WriteObjectStart();
if (tagsListValue != null && tagsListValue.IsSetKey())
{
writer.WritePropertyName("Key");
writer.Write(tagsListValue.Key);
}
if (tagsListValue != null && tagsListValue.IsSetValue())
{
writer.WritePropertyName("Value");
writer.Write(tagsListValue.Value);
}
writer.WriteObjectEnd();
}
writer.WriteArrayEnd();
}
writer.WriteObjectEnd();
string snippet = stringWriter.ToString();
request.Content = System.Text.Encoding.UTF8.GetBytes(snippet);
}
return(request);
}
public IRequest Marshall(RunJobFlowRequest runJobFlowRequest)
{
IRequest request = new DefaultRequest(runJobFlowRequest, "AmazonElasticMapReduce");
request.Parameters.Add("Action", "RunJobFlow");
request.Parameters.Add("Version", "2009-03-31");
if (runJobFlowRequest != null && runJobFlowRequest.IsSetName())
{
request.Parameters.Add("Name", StringUtils.FromString(runJobFlowRequest.Name));
}
if (runJobFlowRequest != null && runJobFlowRequest.IsSetLogUri())
{
request.Parameters.Add("LogUri", StringUtils.FromString(runJobFlowRequest.LogUri));
}
if (runJobFlowRequest != null && runJobFlowRequest.IsSetAdditionalInfo())
{
request.Parameters.Add("AdditionalInfo", StringUtils.FromString(runJobFlowRequest.AdditionalInfo));
}
if (runJobFlowRequest != null && runJobFlowRequest.IsSetAmiVersion())
{
request.Parameters.Add("AmiVersion", StringUtils.FromString(runJobFlowRequest.AmiVersion));
}
if (runJobFlowRequest != null)
{
JobFlowInstancesConfig instances = runJobFlowRequest.Instances;
if (instances != null && instances.IsSetMasterInstanceType())
{
request.Parameters.Add("Instances.MasterInstanceType", StringUtils.FromString(instances.MasterInstanceType));
}
if (instances != null && instances.IsSetSlaveInstanceType())
{
request.Parameters.Add("Instances.SlaveInstanceType", StringUtils.FromString(instances.SlaveInstanceType));
}
if (instances != null && instances.IsSetInstanceCount())
{
request.Parameters.Add("Instances.InstanceCount", StringUtils.FromInt(instances.InstanceCount));
}
if (instances != null)
{
List <InstanceGroupConfig> instanceGroupsList = instances.InstanceGroups;
int instanceGroupsListIndex = 1;
foreach (InstanceGroupConfig instanceGroupsListValue in instanceGroupsList)
{
if (instanceGroupsListValue != null && instanceGroupsListValue.IsSetName())
{
request.Parameters.Add("Instances.InstanceGroups.member." + instanceGroupsListIndex + ".Name", StringUtils.FromString(instanceGroupsListValue.Name));
}
if (instanceGroupsListValue != null && instanceGroupsListValue.IsSetMarket())
{
request.Parameters.Add("Instances.InstanceGroups.member." + instanceGroupsListIndex + ".Market", StringUtils.FromString(instanceGroupsListValue.Market));
}
if (instanceGroupsListValue != null && instanceGroupsListValue.IsSetInstanceRole())
{
request.Parameters.Add("Instances.InstanceGroups.member." + instanceGroupsListIndex + ".InstanceRole", StringUtils.FromString(instanceGroupsListValue.InstanceRole));
}
if (instanceGroupsListValue != null && instanceGroupsListValue.IsSetBidPrice())
{
request.Parameters.Add("Instances.InstanceGroups.member." + instanceGroupsListIndex + ".BidPrice", StringUtils.FromString(instanceGroupsListValue.BidPrice));
}
if (instanceGroupsListValue != null && instanceGroupsListValue.IsSetInstanceType())
{
request.Parameters.Add("Instances.InstanceGroups.member." + instanceGroupsListIndex + ".InstanceType", StringUtils.FromString(instanceGroupsListValue.InstanceType));
}
if (instanceGroupsListValue != null && instanceGroupsListValue.IsSetInstanceCount())
{
request.Parameters.Add("Instances.InstanceGroups.member." + instanceGroupsListIndex + ".InstanceCount", StringUtils.FromInt(instanceGroupsListValue.InstanceCount));
}
instanceGroupsListIndex++;
}
}
if (instances != null && instances.IsSetEc2KeyName())
{
request.Parameters.Add("Instances.Ec2KeyName", StringUtils.FromString(instances.Ec2KeyName));
}
if (instances != null)
{
PlacementType placement = instances.Placement;
if (placement != null && placement.IsSetAvailabilityZone())
{
request.Parameters.Add("Instances.Placement.AvailabilityZone", StringUtils.FromString(placement.AvailabilityZone));
}
}
if (instances != null && instances.IsSetKeepJobFlowAliveWhenNoSteps())
{
request.Parameters.Add("Instances.KeepJobFlowAliveWhenNoSteps", StringUtils.FromBool(instances.KeepJobFlowAliveWhenNoSteps));
}
if (instances != null && instances.IsSetTerminationProtected())
{
request.Parameters.Add("Instances.TerminationProtected", StringUtils.FromBool(instances.TerminationProtected));
}
if (instances != null && instances.IsSetHadoopVersion())
{
request.Parameters.Add("Instances.HadoopVersion", StringUtils.FromString(instances.HadoopVersion));
}
if (instances != null && instances.IsSetEc2SubnetId())
{
request.Parameters.Add("Instances.Ec2SubnetId", StringUtils.FromString(instances.Ec2SubnetId));
}
}
if (runJobFlowRequest != null)
{
List <StepConfig> stepsList = runJobFlowRequest.Steps;
int stepsListIndex = 1;
foreach (StepConfig stepsListValue in stepsList)
{
if (stepsListValue != null && stepsListValue.IsSetName())
{
request.Parameters.Add("Steps.member." + stepsListIndex + ".Name", StringUtils.FromString(stepsListValue.Name));
}
if (stepsListValue != null && stepsListValue.IsSetActionOnFailure())
{
request.Parameters.Add("Steps.member." + stepsListIndex + ".ActionOnFailure", StringUtils.FromString(stepsListValue.ActionOnFailure));
}
if (stepsListValue != null)
{
HadoopJarStepConfig hadoopJarStep = stepsListValue.HadoopJarStep;
if (hadoopJarStep != null)
{
List <KeyValue> propertiesList = hadoopJarStep.Properties;
int propertiesListIndex = 1;
foreach (KeyValue propertiesListValue in propertiesList)
{
if (propertiesListValue != null && propertiesListValue.IsSetKey())
{
request.Parameters.Add("Steps.member." + stepsListIndex + ".HadoopJarStep.Properties.member." + propertiesListIndex + ".Key", StringUtils.FromString(propertiesListValue.Key));
}
if (propertiesListValue != null && propertiesListValue.IsSetValue())
{
request.Parameters.Add("Steps.member." + stepsListIndex + ".HadoopJarStep.Properties.member." + propertiesListIndex + ".Value", StringUtils.FromString(propertiesListValue.Value));
}
propertiesListIndex++;
}
}
if (hadoopJarStep != null && hadoopJarStep.IsSetJar())
{
request.Parameters.Add("Steps.member." + stepsListIndex + ".HadoopJarStep.Jar", StringUtils.FromString(hadoopJarStep.Jar));
}
if (hadoopJarStep != null && hadoopJarStep.IsSetMainClass())
{
request.Parameters.Add("Steps.member." + stepsListIndex + ".HadoopJarStep.MainClass", StringUtils.FromString(hadoopJarStep.MainClass));
}
if (hadoopJarStep != null)
{
List <string> argsList = hadoopJarStep.Args;
int argsListIndex = 1;
foreach (string argsListValue in argsList)
{
request.Parameters.Add("Steps.member." + stepsListIndex + ".HadoopJarStep.Args.member." + argsListIndex, StringUtils.FromString(argsListValue));
argsListIndex++;
}
}
}
stepsListIndex++;
}
}
if (runJobFlowRequest != null)
{
List <BootstrapActionConfig> bootstrapActionsList = runJobFlowRequest.BootstrapActions;
int bootstrapActionsListIndex = 1;
foreach (BootstrapActionConfig bootstrapActionsListValue in bootstrapActionsList)
{
if (bootstrapActionsListValue != null && bootstrapActionsListValue.IsSetName())
{
request.Parameters.Add("BootstrapActions.member." + bootstrapActionsListIndex + ".Name", StringUtils.FromString(bootstrapActionsListValue.Name));
}
if (bootstrapActionsListValue != null)
{
ScriptBootstrapActionConfig scriptBootstrapAction = bootstrapActionsListValue.ScriptBootstrapAction;
if (scriptBootstrapAction != null && scriptBootstrapAction.IsSetPath())
{
request.Parameters.Add("BootstrapActions.member." + bootstrapActionsListIndex + ".ScriptBootstrapAction.Path", StringUtils.FromString(scriptBootstrapAction.Path));
}
if (scriptBootstrapAction != null)
{
List <string> argsList = scriptBootstrapAction.Args;
int argsListIndex = 1;
foreach (string argsListValue in argsList)
{
request.Parameters.Add("BootstrapActions.member." + bootstrapActionsListIndex + ".ScriptBootstrapAction.Args.member." + argsListIndex, StringUtils.FromString(argsListValue));
argsListIndex++;
}
}
}
bootstrapActionsListIndex++;
}
}
if (runJobFlowRequest != null)
{
List <string> supportedProductsList = runJobFlowRequest.SupportedProducts;
int supportedProductsListIndex = 1;
foreach (string supportedProductsListValue in supportedProductsList)
{
request.Parameters.Add("SupportedProducts.member." + supportedProductsListIndex, StringUtils.FromString(supportedProductsListValue));
supportedProductsListIndex++;
}
}
return(request);
}
private void OnJobFlowInstancesConfigCreated(object sender, JobFlowInstancesConfig instancesConfig)
{
this.result.Instances = instancesConfig;
}
public void Visit(JobFlow jobFlow)
{
if (String.IsNullOrEmpty(jobFlow.Name))
{
throw new InvalidOperationException(Resources.E_JobFlowNameIsMissing);
}
if (String.IsNullOrEmpty(jobFlow.LogUri))
{
throw new InvalidOperationException(Resources.E_JobFlowLogUriIsMissing);
}
if (String.IsNullOrEmpty(jobFlow.AmiVersion))
{
throw new InvalidOperationException(Resources.E_JobFlowAmiVersionIsMissing);
}
if (String.IsNullOrEmpty(jobFlow.Ec2KeyName))
{
throw new InvalidOperationException(Resources.E_JobFlowEc2KeyNameIsMissing);
}
if (String.IsNullOrEmpty(jobFlow.HadoopVersion))
{
throw new InvalidOperationException(Resources.E_JobFlowHadoopVersionIsMissing);
}
if (String.IsNullOrEmpty(jobFlow.MasterInstanceType))
{
throw new InvalidOperationException(Resources.E_JobFlowMasterInstanceTypeIsMissing);
}
if (String.IsNullOrEmpty(jobFlow.SlaveInstanceType))
{
throw new InvalidOperationException(Resources.E_JobFlowSlaveInstanceTypeIsMissing);
}
if (jobFlow.InstanceCount <= 0)
{
throw new InvalidOperationException(Resources.E_JobFlowInstanceCountShouldBePositive);
}
RunJobFlowRequest jobFlowRequest = new RunJobFlowRequest();
jobFlowRequest.Name = this.settings.FillPlaceHolders(jobFlow.Name);
jobFlowRequest.LogUri = this.settings.FillPlaceHolders(jobFlow.LogUri);
jobFlowRequest.JobFlowRole = this.settings.FillPlaceHolders(jobFlow.JobFlowRole);
jobFlowRequest.AmiVersion = this.settings.FillPlaceHolders(jobFlow.AmiVersion);
jobFlowRequest.AdditionalInfo = this.settings.FillPlaceHolders(jobFlow.AdditionalInfo);
if (this.OnRunJobFlowRequestCreated != null)
{
this.OnRunJobFlowRequestCreated(this, jobFlowRequest);
}
JobFlowInstancesConfig instancesConfig = new JobFlowInstancesConfig();
instancesConfig.Ec2KeyName = this.settings.FillPlaceHolders(jobFlow.Ec2KeyName);
instancesConfig.HadoopVersion = this.settings.FillPlaceHolders(jobFlow.HadoopVersion);
instancesConfig.KeepJobFlowAliveWhenNoSteps = jobFlow.KeepJobFlowAliveWhenNoSteps;
instancesConfig.TerminationProtected = jobFlow.TerminationProtected;
instancesConfig.MasterInstanceType = this.settings.FillPlaceHolders(jobFlow.MasterInstanceType);
instancesConfig.SlaveInstanceType = this.settings.FillPlaceHolders(jobFlow.SlaveInstanceType);
instancesConfig.InstanceCount = jobFlow.InstanceCount;
if (this.OnJobFlowInstancesConfigCreated != null)
{
this.OnJobFlowInstancesConfigCreated(this, instancesConfig);
}
}
void visitor_OnJobFlowInstancesConfigCreated(object sender, JobFlowInstancesConfig e)
{
this.baseCheck(sender);
this.instanceConfigList.Add(e);
}
