public void VisitHBaseRestoreStep()
{
//Init args
HBaseRestoreStep hBaseRestoreStep = new HBaseRestoreStep();
hBaseRestoreStep.HBaseJarPath = "{hbaseJar}";
hBaseRestoreStep.RestorePath = "{myBucket}/hbase/restore";
//Expectations
List <string> expectedArgs = new List <string>()
{
"--restore", "--backup-dir", "s3://myBucket/hbase/restore"
};
//Init settings
IBuilderSettings settings = BuildRequestVisitorTest.GetSettings();
//Init visitor
BuildRequestVisitor visitor = new BuildRequestVisitor(settings);
VisitorSubscriber visitorSubscriber = new VisitorSubscriber(visitor);
//Action
hBaseRestoreStep.Accept(visitor);
//Verify
Assert.AreEqual(1, visitorSubscriber.TotalObjCount, "Unexpected number of objects created");
StepConfig actual = visitorSubscriber.stepList[0];
Assert.AreEqual("Restore HBase", actual.Name, "Unexpected Name");
Assert.AreEqual(ActionOnFailure.TERMINATE_JOB_FLOW, actual.ActionOnFailure, "Unexpected ActionOnFailure");
Assert.AreEqual("/home/hadoop/lib/2.2.0/hbase-0.94.7.jar", actual.HadoopJarStep.Jar, "Unexpected Jar");
Assert.AreEqual("emr.hbase.backup.Main", actual.HadoopJarStep.MainClass, "Unexpected MainClass");
Assert.IsTrue(expectedArgs.SequenceEqual(actual.HadoopJarStep.Args), "Unexpected args list");
}
public RunJobFlowRequestBuilder(IBuilderSettings settings)
{
this.visitor = new BuildRequestVisitor(settings);
this.visitor.OnRunJobFlowRequestCreated += this.OnRunJobFlowRequestCreated;
this.visitor.OnJobFlowInstancesConfigCreated += this.OnJobFlowInstancesConfigCreated;
this.visitor.OnTagCreated += this.OnTagCreated;
this.visitor.OnBootstrapActionConfigCreated += this.OnBootstrapActionConfigCreated;
this.visitor.OnStepConfigCreated += this.OnStepConfigCreated;
}
public async Task <bool> Run()
{
//Create dependencies
IBuilderSettings settings = this.CreateSettings();
AmazonElasticMapReduceClient emrClient = this.CreateEmrClient();
IEmrJobLogger emrJobLogger = new EmrJobLogger();
IEmrJobStateChecker emrJobStateChecker = new EmrJobStateChecker();
DemoEmrActivitiesEnumerator activitiesIterator = new DemoEmrActivitiesEnumerator();
using (EmrActivitiesRunner emrRunner = new EmrActivitiesRunner(emrJobLogger, emrJobStateChecker, emrClient, settings, activitiesIterator))
{
//explicitly set an existing jobFlowId, if you want to work with an existing job
//emrRunner.JobFlowId = "j-36G3NHTVEP1Q7";
return(await emrRunner.Start());
}
}
public void VisitHBaseConfigNullArgs()
{
//Init args
HBaseConfig hBaseConfig = new HBaseConfig()
{
IfStart = true
};
hBaseConfig.JarPath = "/home/hadoop/lib/hbase-0.94.7.jar";
//Expectations
List <string> expectedStartArgs = new List <string>()
{
"--start-master"
};
//Init settings
IBuilderSettings settings = BuildRequestVisitorTest.GetSettings();
//Init visitor
BuildRequestVisitor visitor = new BuildRequestVisitor(settings);
VisitorSubscriber visitorSubscriber = new VisitorSubscriber(visitor);
//Action
hBaseConfig.Accept(visitor);
//Verify
Assert.AreEqual(2, visitorSubscriber.TotalObjCount, "Unexpected number of objects created");
// BootstrapAction 1: Install HBase
BootstrapActionConfig actualBootstrapAction = visitorSubscriber.bootstrapActionList[0];
Assert.AreEqual("Install HBase", actualBootstrapAction.Name, "Unexpected Name");
Assert.AreEqual("s3://elasticmapreduce/bootstrap-actions/setup-hbase", actualBootstrapAction.ScriptBootstrapAction.Path, "Unexpected ScriptBootstrapAction.Path");
Assert.IsNull(actualBootstrapAction.ScriptBootstrapAction.Args, "Unexpected args list");
//Step : Start HBase
StepConfig actualStep = visitorSubscriber.stepList[0];
Assert.AreEqual("Start HBase", actualStep.Name, "Unexpected Name");
Assert.AreEqual(ActionOnFailure.TERMINATE_JOB_FLOW, actualStep.ActionOnFailure, "Unexpected ActionOnFailure");
Assert.AreEqual("/home/hadoop/lib/hbase-0.94.7.jar", actualStep.HadoopJarStep.Jar, "Unexpected Jar");
Assert.AreEqual("emr.hbase.backup.Main", actualStep.HadoopJarStep.MainClass, "Unexpected MainClass");
Assert.IsTrue(expectedStartArgs.SequenceEqual(actualStep.HadoopJarStep.Args), "Unexpected args list");
}
public IBuilder Create()
{
IBuilder pattern = null;
int index = (int)UnityEngine.Random.value * _presets.builders.Count;
IBuilderSettings toCreateSettings = _presets.builders[index];
switch (toCreateSettings.Id)
{
case (int)BuilderId.GROUPED_BUILDER:
pattern = _container.Instantiate <GroupedObstacleBuilder>(new object[] { toCreateSettings });
break;
case (int)BuilderId.LINEAR_BUILDER:
pattern = _container.Instantiate <LinearBuilder>(new object[] { toCreateSettings });
break;
}
return(pattern);
}
/// <summary>
/// Send a request to EMR service to add new step/steps
/// </summary>
/// <param name="emrClient">EMR Client to make requests to the Amazon EMR Service</param>
/// <param name="settings">Settings to replace placeholders</param>
/// <param name="jobFlowId">Existing jobflow Id, can be null for the new job.</param>
/// <returns>JobFlow Id, if request failed -> returns null</returns>
public override async Task <string> SendAsync(IAmazonElasticMapReduce emrClient, IBuilderSettings settings, string jobFlowId)
{
AddJobFlowStepsRequestBuilder builder = new AddJobFlowStepsRequestBuilder(settings);
AddJobFlowStepsRequest request = builder.Build(jobFlowId, this.steps);
AddJobFlowStepsResponse response = await emrClient.AddJobFlowStepsAsync(request);
return(this.IsOk(response) ? jobFlowId : null);
}
/// <summary>
/// Send a request to EMR service to start and configure a new job
/// </summary>
/// <param name="emrClient">EMR Client to make requests to the Amazon EMR Service</param>
/// <param name="settings">Settings to replace placeholders</param>
/// <param name="jobFlowId">Existing jobflow Id, can be null for the new job.</param>
/// <returns>JobFlow Id, if request failed -> returns null</returns>
public override async Task <string> SendAsync(IAmazonElasticMapReduce emrClient, IBuilderSettings settings, string jobFlowId)
{
RunJobFlowRequestBuilder builder = new RunJobFlowRequestBuilder(settings);
RunJobFlowRequest request = builder.Build(this.jobFlow);
RunJobFlowResponse response = await emrClient.RunJobFlowAsync(request);
if (!this.IsOk(response))
{
return(null);
}
return(response.JobFlowId);
}
/// <summary>
/// Constructor for injecting dependencies
/// </summary>
/// <param name="emrJobLogger">Instantiated object to log information about the EMR Job</param>
/// <param name="emrJobStateChecker">Instantiated object to check the current state of the EMR Job</param>
/// <param name="settings">Settings to replace placeholders</param>
/// <param name="emrClient">Instantiated EMR Client to make requests to the Amazon EMR Service</param>
/// <param name="swfClient">Instantiated SWF Client to make requests to the Amazon SWF Service</param>
/// <param name="swfConfiguration">Instantiated object that provides configuration info for the current SWF</param>
public SwfActivitiesRunner(IEmrJobLogger emrJobLogger, IEmrJobStateChecker emrJobStateChecker, IBuilderSettings settings, IAmazonElasticMapReduce emrClient, IAmazonSimpleWorkflow swfClient, ISwfConfiguration swfConfiguration)
{
this.EmrJobLogger = emrJobLogger;
this.EmrJobStateChecker = emrJobStateChecker;
this.Settings = settings;
this.EmrClient = emrClient;
this.SwfClient = swfClient;
this.SwfConfiguration = swfConfiguration;
}
/// <summary>
/// Send a request to EMR service to terminate job
/// </summary>
/// <param name="emrClient">EMR Client to make requests to the Amazon EMR Service</param>
/// <param name="settings">Settings to replace placeholders</param>
/// <param name="jobFlowId">Existing jobflow Id, can be null for the new job.</param>
/// <returns>JobFlow Id, if request failed -> returns null</returns>
public override async Task <string> SendAsync(IAmazonElasticMapReduce emrClient, IBuilderSettings settings, string jobFlowId)
{
SetTerminationProtectionRequest setTerminationProtectionRequest = new SetTerminationProtectionRequest();
setTerminationProtectionRequest.JobFlowIds = new List <string> {
jobFlowId
};
setTerminationProtectionRequest.TerminationProtected = false;
AmazonWebServiceResponse response = await emrClient.SetTerminationProtectionAsync(setTerminationProtectionRequest);
TerminateJobFlowsRequest terminateJobRequest = new TerminateJobFlowsRequest();
terminateJobRequest.JobFlowIds = new List <string> {
jobFlowId
};
response = await emrClient.TerminateJobFlowsAsync(terminateJobRequest);
return(this.IsOk(response) ? jobFlowId : null);
}
/// <summary> /// Send a request to EMR service to do some job /// </summary> /// <param name="emrClient">EMR Client to make requests to the Amazon EMR Service</param> /// <param name="settings">Settings to replace placeholders</param> /// <param name="jobFlowId">Existing jobflow Id, can be null for the new job.</param> /// <returns>JobFlow Id, if request failed -> returns null</returns> public abstract Task <string> SendAsync(IAmazonElasticMapReduce emrClient, IBuilderSettings settings, string jobFlowId);
public AddJobFlowStepsRequestBuilder(IBuilderSettings settings)
{
this.visitor = new BuildRequestVisitor(settings);
this.visitor.OnStepConfigCreated += this.OnStepConfigCreated;
}
public BuildRequestVisitor(IBuilderSettings settings)
{
this.settings = settings;
}
/// <summary>
/// Constructor for injecting dependencies
/// </summary>
/// <param name="emrJobLogger">Instantiated object to log information about the EMR Job</param>
/// <param name="emrJobStateChecker">Instantiated object to check the current state of the EMR Job</param>
/// <param name="emrClient">Instantiated EMR Client to make requests to the Amazon EMR Service</param>
/// <param name="settings">Settings to replace placeholders</param>
/// <param name="emrActivitiesEnumerator">Iterator through the job flow's activities</param>
public EmrActivitiesRunner(IEmrJobLogger emrJobLogger, IEmrJobStateChecker emrJobStateChecker, IAmazonElasticMapReduce emrClient, IBuilderSettings settings, EmrActivitiesIteratorBase emrActivitiesEnumerator)
{
this.hasErrors = false;
this.Settings = settings;
this.EmrClient = emrClient;
this.EmrJobLogger = emrJobLogger;
this.EmrJobStateChecker = emrJobStateChecker;
this.EmrActivitiesIterator = emrActivitiesEnumerator;
}
