/// <summary>
/// Send a request to the EMR service to get the latest state of the job
/// </summary>
/// <param name="emrClient">Instantiated EMR Client to make requests to the Amazon EMR Service</param>
/// <param name="jobFlowId">EMR Job flow id</param>
/// <returns>Current state of the EMR Job</returns>
public async Task <EmrActivityInfo> CheckAsync(IAmazonElasticMapReduce emrClient, String jobFlowId)
{
//Read job state
DescribeJobFlowsRequest request = new DescribeJobFlowsRequest();
request.JobFlowIds = new List <string>()
{
jobFlowId
};
DescribeJobFlowsResponse response = await emrClient.DescribeJobFlowsAsync(request);
if (response.HttpStatusCode != HttpStatusCode.OK)
{
return new EmrActivityInfo()
{
CurrentState = EmrActivityState.Failed
}
}
;
//Map job state into Completed, Failed or Running
JobFlowDetail jobFlowDetail = response.JobFlows[0];
EmrActivityState activityState = EmrJobStateChecker.GetState(jobFlowDetail);
return(new EmrActivityInfo()
{
JobFlowDetail = jobFlowDetail, CurrentState = activityState
});
}
public static void InvokeDescribeJobFlows()
{
NameValueCollection appConfig = ConfigurationManager.AppSettings;
// Print the number of Amazon SimpleDB domains.
IAmazonElasticMapReduce emr = AWSClientFactory.CreateAmazonElasticMapReduceClient(RegionEndpoint.USWest2);
try
{
DescribeJobFlowsResponse emrResponse = emr.DescribeJobFlows(new DescribeJobFlowsRequest());
int numFlows = 0;
numFlows = emrResponse.JobFlows.Count;
Console.WriteLine("You have " + numFlows + " Amazon Elastic MapReduce job flow(s).");
}
catch (AmazonElasticMapReduceException ex)
{
if (ex.ErrorCode.Equals("OptInRequired"))
{
Console.WriteLine("You are not signed for Amazon Elastic MapReduce.");
Console.WriteLine("You can sign up at http://aws.amazon.com/elasticmapreduce.");
}
else
{
Console.WriteLine("Caught Exception: " + ex.Message);
Console.WriteLine("Response Status Code: " + ex.StatusCode);
Console.WriteLine("Error Code: " + ex.ErrorCode);
Console.WriteLine("Error Type: " + ex.ErrorType);
Console.WriteLine("Request ID: " + ex.RequestId);
}
}
Console.WriteLine();
}
/// <summary>
/// Send a request to EMR service to add new step/steps
/// </summary>
/// <param name="emrClient">EMR Client to make requests to the Amazon EMR Service</param>
/// <param name="settings">Settings to replace placeholders</param>
/// <param name="jobFlowId">Existing jobflow Id, can be null for the new job.</param>
/// <returns>JobFlow Id, if request failed -> returns null</returns>
public override async Task <string> SendAsync(IAmazonElasticMapReduce emrClient, IBuilderSettings settings, string jobFlowId)
{
AddJobFlowStepsRequestBuilder builder = new AddJobFlowStepsRequestBuilder(settings);
AddJobFlowStepsRequest request = builder.Build(jobFlowId, this.steps);
AddJobFlowStepsResponse response = await emrClient.AddJobFlowStepsAsync(request);
return(this.IsOk(response) ? jobFlowId : null);
}
/// <summary>
/// Send a request to EMR service to start and configure a new job
/// </summary>
/// <param name="emrClient">EMR Client to make requests to the Amazon EMR Service</param>
/// <param name="settings">Settings to replace placeholders</param>
/// <param name="jobFlowId">Existing jobflow Id, can be null for the new job.</param>
/// <returns>JobFlow Id, if request failed -> returns null</returns>
public override async Task <string> SendAsync(IAmazonElasticMapReduce emrClient, IBuilderSettings settings, string jobFlowId)
{
RunJobFlowRequestBuilder builder = new RunJobFlowRequestBuilder(settings);
RunJobFlowRequest request = builder.Build(this.jobFlow);
RunJobFlowResponse response = await emrClient.RunJobFlowAsync(request);
if (!this.IsOk(response))
{
return(null);
}
return(response.JobFlowId);
}
public static async Task <RunJobFlowResponse> RunJob(IAmazonElasticMapReduce emrClient, JobConfiguration configuration)
{
RunJobFlowRequest request = new RunJobFlowRequest();
request.Name = configuration.ClusterName;
request.ReleaseLabel = configuration.ReleaseLabel;
request.Applications = configuration.Applications.Select(app => new Amazon.ElasticMapReduce.Model.Application()
{
Name = app.Name
}).ToList();
request.Instances = new JobFlowInstancesConfig();
request.Instances.Ec2KeyName = configuration.EC2KeyName;
request.Instances.InstanceCount = configuration.InstanceCount;
request.Instances.KeepJobFlowAliveWhenNoSteps = configuration.KeepJobAlive;
request.Instances.MasterInstanceType = configuration.MasterInstanceType;
request.Instances.SlaveInstanceType = configuration.SlaveInstanceType;
request.ServiceRole = configuration.ServiceRole;
request.JobFlowRole = configuration.JobFlowRole;
request.BootstrapActions = configuration.BootstrapActions.Select(bootstrap => new Amazon.ElasticMapReduce.Model.BootstrapActionConfig()
{
Name = bootstrap.Name,
ScriptBootstrapAction = new ScriptBootstrapActionConfig()
{
Path = bootstrap.ScriptBootstrapAction.Path,
Args = new List <string>(bootstrap.ScriptBootstrapAction.Args)
}
}).ToList();
request.Steps = configuration.StepConfigurations.Select(s => new Amazon.ElasticMapReduce.Model.StepConfig()
{
Name = s.Name,
ActionOnFailure = ParseActionOnFailure(s.ActionOnFailure),
HadoopJarStep = new HadoopJarStepConfig()
{
Jar = s.HadoopJarStep.Jar,
Args = new List <string>(s.HadoopJarStep.Args)
}
}).ToList();
request.LogUri = configuration.LogUri;
return(await emrClient.RunJobFlowAsync(request));
}
/// <summary>
/// Send a request to EMR service to terminate job
/// </summary>
/// <param name="emrClient">EMR Client to make requests to the Amazon EMR Service</param>
/// <param name="settings">Settings to replace placeholders</param>
/// <param name="jobFlowId">Existing jobflow Id, can be null for the new job.</param>
/// <returns>JobFlow Id, if request failed -> returns null</returns>
public override async Task <string> SendAsync(IAmazonElasticMapReduce emrClient, IBuilderSettings settings, string jobFlowId)
{
SetTerminationProtectionRequest setTerminationProtectionRequest = new SetTerminationProtectionRequest();
setTerminationProtectionRequest.JobFlowIds = new List <string> {
jobFlowId
};
setTerminationProtectionRequest.TerminationProtected = false;
AmazonWebServiceResponse response = await emrClient.SetTerminationProtectionAsync(setTerminationProtectionRequest);
TerminateJobFlowsRequest terminateJobRequest = new TerminateJobFlowsRequest();
terminateJobRequest.JobFlowIds = new List <string> {
jobFlowId
};
response = await emrClient.TerminateJobFlowsAsync(terminateJobRequest);
return(this.IsOk(response) ? jobFlowId : null);
}
private OperationResult EstablishClient(AddonManifest manifest, EMRDeveloperOptions devOptions, out IAmazonElasticMapReduce client)
{
OperationResult result;
bool requireCreds;
var accessKey = manifest.ProvisioningUsername;
var secretAccessKey = manifest.ProvisioningPassword;
var prop = manifest.Properties.First(p => p.Key.Equals("requireDevCredentials", StringComparison.InvariantCultureIgnoreCase));
//jobid = null;
if (bool.TryParse(prop.Value, out requireCreds) && requireCreds)
{
if (!ValidateDevCreds(devOptions))
{
client = null;
result = new OperationResult()
{
IsSuccess = false,
EndUserMessage = "The add on requires that developer credentials are specified but none were provided."
};
return(result);
}
accessKey = devOptions.AccessKey;
secretAccessKey = devOptions.SecretAccessKey;
}
AWSCredentials credentials = new BasicAWSCredentials(accessKey, secretAccessKey);
client = AWSClientFactory.CreateAmazonElasticMapReduceClient(credentials, RegionEndpoint.USEast1);
//jobid = job.JobFlowId;
result = new OperationResult()
{
IsSuccess = true
};
return(result);
}
public TaskRunnerController(IAmazonElasticMapReduce emrClient, ITaskConfigurationService taskConfigurationService)
{
EMRClient = emrClient;
TaskConfigurationService = taskConfigurationService;
}
internal ListReleaseLabelsPaginator(IAmazonElasticMapReduce client, ListReleaseLabelsRequest request)
{
this._client = client;
this._request = request;
}
internal ListBootstrapActionsPaginator(IAmazonElasticMapReduce client, ListBootstrapActionsRequest request)
{
this._client = client;
this._request = request;
}
internal ListSecurityConfigurationsPaginator(IAmazonElasticMapReduce client, ListSecurityConfigurationsRequest request)
{
this._client = client;
this._request = request;
}
/// <summary> /// Send a request to EMR service to do some job /// </summary> /// <param name="emrClient">EMR Client to make requests to the Amazon EMR Service</param> /// <param name="settings">Settings to replace placeholders</param> /// <param name="jobFlowId">Existing jobflow Id, can be null for the new job.</param> /// <returns>JobFlow Id, if request failed -> returns null</returns> public abstract Task <string> SendAsync(IAmazonElasticMapReduce emrClient, IBuilderSettings settings, string jobFlowId);
private OperationResult EstablishClient(AddonManifest manifest, EMRDeveloperOptions devOptions, out IAmazonElasticMapReduce client)
{
OperationResult result;
bool requireCreds;
var accessKey = manifest.ProvisioningUsername;
var secretAccessKey = manifest.ProvisioningPassword;
var prop = manifest.Properties.First(p => p.Key.Equals("requireDevCredentials", StringComparison.InvariantCultureIgnoreCase));
//jobid = null;
if (bool.TryParse(prop.Value, out requireCreds) && requireCreds)
{
if (!ValidateDevCreds(devOptions))
{
client = null;
result = new OperationResult()
{
IsSuccess = false,
EndUserMessage = "The add on requires that developer credentials are specified but none were provided."
};
return result;
}
accessKey = devOptions.AccessKey;
secretAccessKey = devOptions.SecretAccessKey;
}
AWSCredentials credentials = new BasicAWSCredentials(accessKey, secretAccessKey);
client = AWSClientFactory.CreateAmazonElasticMapReduceClient(credentials, RegionEndpoint.USEast1);
//jobid = job.JobFlowId;
result = new OperationResult()
{
IsSuccess = true
};
return result;
}
protected override void ProcessRecord()
{
base.ProcessRecord();
Client = CreateClient(_CurrentCredentials, _RegionEndpoint);
}
public static async Task <ListClustersResponse> ListJobs(IAmazonElasticMapReduce emrClient)
{
return(await emrClient.ListClustersAsync());
}
internal ListStudioSessionMappingsPaginator(IAmazonElasticMapReduce client, ListStudioSessionMappingsRequest request)
{
this._client = client;
this._request = request;
}
/// <summary>
/// Constructor for injecting dependencies
/// </summary>
/// <param name="emrJobLogger">Instantiated object to log information about the EMR Job</param>
/// <param name="emrJobStateChecker">Instantiated object to check the current state of the EMR Job</param>
/// <param name="settings">Settings to replace placeholders</param>
/// <param name="emrClient">Instantiated EMR Client to make requests to the Amazon EMR Service</param>
/// <param name="swfClient">Instantiated SWF Client to make requests to the Amazon SWF Service</param>
/// <param name="swfConfiguration">Instantiated object that provides configuration info for the current SWF</param>
public SwfActivitiesRunner(IEmrJobLogger emrJobLogger, IEmrJobStateChecker emrJobStateChecker, IBuilderSettings settings, IAmazonElasticMapReduce emrClient, IAmazonSimpleWorkflow swfClient, ISwfConfiguration swfConfiguration)
{
this.EmrJobLogger = emrJobLogger;
this.EmrJobStateChecker = emrJobStateChecker;
this.Settings = settings;
this.EmrClient = emrClient;
this.SwfClient = swfClient;
this.SwfConfiguration = swfConfiguration;
}
internal ListInstanceGroupsPaginator(IAmazonElasticMapReduce client, ListInstanceGroupsRequest request)
{
this._client = client;
this._request = request;
}
internal ElasticMapReducePaginatorFactory(IAmazonElasticMapReduce client)
{
this.client = client;
}
internal ListNotebookExecutionsPaginator(IAmazonElasticMapReduce client, ListNotebookExecutionsRequest request)
{
this._client = client;
this._request = request;
}
internal ListClustersPaginator(IAmazonElasticMapReduce client, ListClustersRequest request)
{
this._client = client;
this._request = request;
}
private Amazon.ElasticMapReduce.Model.ListStepsResponse CallAWSServiceOperation(IAmazonElasticMapReduce client, Amazon.ElasticMapReduce.Model.ListStepsRequest request)
{
Utils.Common.WriteVerboseEndpointMessage(this, client.Config, "Amazon Elastic MapReduce", "ListSteps");
try
{
#if DESKTOP
return(client.ListSteps(request));
#elif CORECLR
return(client.ListStepsAsync(request).GetAwaiter().GetResult());
#else
#error "Unknown build edition"
#endif
}
catch (AmazonServiceException exc)
{
var webException = exc.InnerException as System.Net.WebException;
if (webException != null)
{
throw new Exception(Utils.Common.FormatNameResolutionFailureMessage(client.Config, webException.Message), webException);
}
throw;
}
}
/// <summary>
/// Constructor for injecting dependencies
/// </summary>
/// <param name="emrJobLogger">Instantiated object to log information about the EMR Job</param>
/// <param name="emrJobStateChecker">Instantiated object to check the current state of the EMR Job</param>
/// <param name="emrClient">Instantiated EMR Client to make requests to the Amazon EMR Service</param>
/// <param name="settings">Settings to replace placeholders</param>
/// <param name="emrActivitiesEnumerator">Iterator through the job flow's activities</param>
public EmrActivitiesRunner(IEmrJobLogger emrJobLogger, IEmrJobStateChecker emrJobStateChecker, IAmazonElasticMapReduce emrClient, IBuilderSettings settings, EmrActivitiesIteratorBase emrActivitiesEnumerator)
{
this.hasErrors = false;
this.Settings = settings;
this.EmrClient = emrClient;
this.EmrJobLogger = emrJobLogger;
this.EmrJobStateChecker = emrJobStateChecker;
this.EmrActivitiesIterator = emrActivitiesEnumerator;
}