protected override void ProcessRecord()
{
AmazonElasticMapReduce client = base.GetClient();
Amazon.ElasticMapReduce.Model.RunJobFlowRequest request = new Amazon.ElasticMapReduce.Model.RunJobFlowRequest();
request.Name = this._Name;
request.LogUri = this._LogUri;
request.AdditionalInfo = this._AdditionalInfo;
Amazon.ElasticMapReduce.Model.RunJobFlowResponse response = client.RunJobFlow(request);
base.WriteObject(response.RunJobFlowResult, true);
}
/// <summary>
/// Initiates the asynchronous execution of the RunJobFlow operation.
/// </summary>
///
/// <param name="request">Container for the necessary parameters to execute the RunJobFlow operation.</param>
/// <param name="cancellationToken">
/// A cancellation token that can be used by other objects or threads to receive notice of cancellation.
/// </param>
/// <returns>The task object representing the asynchronous operation.</returns>
public Task<RunJobFlowResponse> RunJobFlowAsync(RunJobFlowRequest request, System.Threading.CancellationToken cancellationToken = default(CancellationToken))
{
var marshaller = new RunJobFlowRequestMarshaller();
var unmarshaller = RunJobFlowResponseUnmarshaller.Instance;
return InvokeAsync<RunJobFlowRequest,RunJobFlowResponse>(request, marshaller,
unmarshaller, cancellationToken);
}
/// <summary>
/// RunJobFlow creates and starts running a new job flow. The job flow will run the steps
/// specified. Once the job flow completes, the cluster is stopped and the HDFS partition
/// is lost. To prevent loss of data, configure the last step of the job flow to store
/// results in Amazon S3. If the <a>JobFlowInstancesConfig</a> <code>KeepJobFlowAliveWhenNoSteps</code>
/// parameter is set to <code>TRUE</code>, the job flow will transition to the WAITING
/// state rather than shutting down once the steps have completed.
///
///
/// <para>
/// For additional protection, you can set the <a>JobFlowInstancesConfig</a> <code>TerminationProtected</code>
/// parameter to <code>TRUE</code> to lock the job flow and prevent it from being terminated
/// by API call, user intervention, or in the event of a job flow error.
/// </para>
///
/// <para>
/// A maximum of 256 steps are allowed in each job flow.
/// </para>
///
/// <para>
/// If your job flow is long-running (such as a Hive data warehouse) or complex, you may
/// require more than 256 steps to process your data. You can bypass the 256-step limitation
/// in various ways, including using the SSH shell to connect to the master node and submitting
/// queries directly to the software running on the master node, such as Hive and Hadoop.
/// For more information on how to do this, go to <a href="http://docs.aws.amazon.com/ElasticMapReduce/latest/DeveloperGuide/AddMoreThan256Steps.html">Add
/// More than 256 Steps to a Job Flow</a> in the <i>Amazon Elastic MapReduce Developer's
/// Guide</i>.
/// </para>
///
/// <para>
/// For long running job flows, we recommend that you periodically store your results.
/// </para>
/// </summary>
/// <param name="request">Container for the necessary parameters to execute the RunJobFlow service method.</param>
///
/// <returns>The response from the RunJobFlow service method, as returned by ElasticMapReduce.</returns>
/// <exception cref="Amazon.ElasticMapReduce.Model.InternalServerErrorException">
/// Indicates that an error occurred while processing the request and that the request
/// was not completed.
/// </exception>
public RunJobFlowResponse RunJobFlow(RunJobFlowRequest request)
{
var marshaller = new RunJobFlowRequestMarshaller();
var unmarshaller = RunJobFlowResponseUnmarshaller.Instance;
return Invoke<RunJobFlowRequest,RunJobFlowResponse>(request, marshaller, unmarshaller);
}
/// <summary>
/// RunJobFlow creates and starts running a new job flow. The job flow will run the steps
/// specified. Once the job flow completes, the cluster is stopped and the HDFS
/// partition is lost. To prevent loss of data, configure the last step of the
/// job flow to store results in Amazon S3. If the <a>JobFlowInstancesConfig</a>
/// <code>KeepJobFlowAliveWhenNoSteps</code> parameter is set to <code>TRUE</code>,
/// the job flow will transition to the WAITING state rather than shutting down
/// once the steps have completed.
///
///
/// <para>
/// For additional protection, you can set the <a>JobFlowInstancesConfig</a>
/// <code>TerminationProtected</code> parameter to <code>TRUE</code> to lock the
/// job flow and prevent it from being terminated by API call, user intervention,
/// or in the event of a job flow error.
/// </para>
///
/// <para>
/// A maximum of 256 steps are allowed in each job flow.
/// </para>
///
/// <para>
/// If your job flow is long-running (such as a Hive data warehouse) or complex, you may
/// require more than 256 steps to process your data. You can bypass the 256-step limitation
/// in various ways, including using the SSH shell to connect to the master node and submitting
/// queries directly to the software running on the master node, such as Hive and Hadoop.
/// For more information on how to do this, go to <a href="http://docs.aws.amazon.com/ElasticMapReduce/latest/DeveloperGuide/AddMoreThan256Steps.html">Add
/// More than 256 Steps to a Job Flow</a> in the <i>Amazon Elastic MapReduce Developer's
/// Guide</i>.
/// </para>
///
/// <para>
/// For long running job flows, we recommend that you periodically store your results.
/// </para>
/// </summary>
/// <param name="request">Container for the necessary parameters to execute the RunJobFlow service method.</param>
///
/// <returns>The response from the RunJobFlow service method, as returned by ElasticMapReduce.</returns>
/// <exception cref="InternalServerErrorException">
/// Indicates that an error occurred while processing the request and that the request
/// was not completed.
/// </exception>
public RunJobFlowResponse RunJobFlow(RunJobFlowRequest request)
{
var task = RunJobFlowAsync(request);
try
{
return task.Result;
}
catch(AggregateException e)
{
ExceptionDispatchInfo.Capture(e.InnerException).Throw();
return null;
}
}
/// <summary>
/// Initiates the asynchronous execution of the RunJobFlow operation.
/// <seealso cref="Amazon.ElasticMapReduce.IAmazonElasticMapReduce.RunJobFlow"/>
/// </summary>
///
/// <param name="request">Container for the necessary parameters to execute the RunJobFlow operation.</param>
/// <param name="cancellationToken">
/// A cancellation token that can be used by other objects or threads to receive notice of cancellation.
/// </param>
/// <returns>The task object representing the asynchronous operation.</returns>
public Task<RunJobFlowResponse> RunJobFlowAsync(RunJobFlowRequest request, CancellationToken cancellationToken = default(CancellationToken))
{
var marshaller = new RunJobFlowRequestMarshaller();
var unmarshaller = RunJobFlowResponseUnmarshaller.GetInstance();
return Invoke<IRequest, RunJobFlowRequest, RunJobFlowResponse>(request, marshaller, unmarshaller, signer, cancellationToken);
}
/// <summary>
/// <para> RunJobFlow creates and starts running a new job flow. The job flow will run the steps specified. Once the job flow completes, the
/// cluster is stopped and the HDFS partition is lost. To prevent loss of data, configure the last step of the job flow to store results in
/// Amazon S3. If the JobFlowInstancesConfig <c>KeepJobFlowAliveWhenNoSteps</c> parameter is set to <c>TRUE</c> , the job flow will transition
/// to the WAITING state rather than shutting down once the steps have completed. </para> <para>For additional protection, you can set the
/// JobFlowInstancesConfig <c>TerminationProtected</c> parameter to <c>TRUE</c> to lock the job flow and prevent it from being terminated by API
/// call, user intervention, or in the event of a job flow error.</para> <para>A maximum of 256 steps are allowed in each job flow.</para>
/// <para>If your job flow is long-running (such as a Hive data warehouse) or complex, you may require more than 256 steps to process your data.
/// You can bypass the 256-step limitation in various ways, including using the SSH shell to connect to the master node and submitting queries
/// directly to the software running on the master node, such as Hive and Hadoop. For more information on how to do this, go to <a href="http://docs.aws.amazon.com/ElasticMapReduce/latest/DeveloperGuide/AddMoreThan256Steps.html">Add More than 256 Steps to a Job Flow</a>
/// in the <i>Amazon Elastic MapReduce Developer's Guide</i> .</para> <para>For long running job flows, we recommend that you periodically store
/// your results.</para>
/// </summary>
///
/// <param name="request">Container for the necessary parameters to execute the RunJobFlow service method on
/// AmazonElasticMapReduce.</param>
///
/// <returns>The response from the RunJobFlow service method, as returned by AmazonElasticMapReduce.</returns>
///
/// <exception cref="T:Amazon.ElasticMapReduce.Model.InternalServerErrorException" />
public RunJobFlowResponse RunJobFlow(RunJobFlowRequest request)
{
var task = RunJobFlowAsync(request);
try
{
return task.Result;
}
catch(AggregateException e)
{
throw e.InnerException;
}
}
/// <summary>
/// Initiates the asynchronous execution of the RunJobFlow operation.
/// </summary>
///
/// <param name="request">Container for the necessary parameters to execute the RunJobFlow operation on AmazonElasticMapReduceClient.</param>
/// <param name="callback">An AsyncCallback delegate that is invoked when the operation completes.</param>
/// <param name="state">A user-defined state object that is passed to the callback procedure. Retrieve this object from within the callback
/// procedure using the AsyncState property.</param>
///
/// <returns>An IAsyncResult that can be used to poll or wait for results, or both; this value is also needed when invoking EndRunJobFlow
/// operation.</returns>
public IAsyncResult BeginRunJobFlow(RunJobFlowRequest request, AsyncCallback callback, object state)
{
var marshaller = new RunJobFlowRequestMarshaller();
var unmarshaller = RunJobFlowResponseUnmarshaller.Instance;
return BeginInvoke<RunJobFlowRequest>(request, marshaller, unmarshaller,
callback, state);
}
IAsyncResult invokeRunJobFlow(RunJobFlowRequest runJobFlowRequest, AsyncCallback callback, object state, bool synchronized)
{
IRequest irequest = new RunJobFlowRequestMarshaller().Marshall(runJobFlowRequest);
var unmarshaller = RunJobFlowResponseUnmarshaller.GetInstance();
AsyncResult result = new AsyncResult(irequest, callback, state, synchronized, signer, unmarshaller);
Invoke(result);
return result;
}
/// <summary>
/// Initiates the asynchronous execution of the RunJobFlow operation.
/// <seealso cref="Amazon.ElasticMapReduce.IAmazonElasticMapReduce.RunJobFlow"/>
/// </summary>
///
/// <param name="runJobFlowRequest">Container for the necessary parameters to execute the RunJobFlow operation on
/// AmazonElasticMapReduce.</param>
/// <param name="callback">An AsyncCallback delegate that is invoked when the operation completes.</param>
/// <param name="state">A user-defined state object that is passed to the callback procedure. Retrieve this object from within the callback
/// procedure using the AsyncState property.</param>
///
/// <returns>An IAsyncResult that can be used to poll or wait for results, or both; this value is also needed when invoking EndRunJobFlow
/// operation.</returns>
public IAsyncResult BeginRunJobFlow(RunJobFlowRequest runJobFlowRequest, AsyncCallback callback, object state)
{
return invokeRunJobFlow(runJobFlowRequest, callback, state, false);
}
/// <summary>
/// <para> RunJobFlow creates and starts running a new job flow. The job flow will run the steps specified. Once the job flow completes, the
/// cluster is stopped and the HDFS partition is lost. To prevent loss of data, configure the last step of the job flow to store results in
/// Amazon S3. If the JobFlowInstancesConfig <c>KeepJobFlowAliveWhenNoSteps</c> parameter is set to <c>TRUE</c> , the job flow will transition
/// to the WAITING state rather than shutting down once the steps have completed. </para> <para>For additional protection, you can set the
/// JobFlowInstancesConfig <c>TerminationProtected</c> parameter to <c>TRUE</c> to lock the job flow and prevent it from being terminated by API
/// call, user intervention, or in the event of a job flow error.</para> <para>A maximum of 256 steps are allowed in each job flow.</para>
/// <para>If your job flow is long-running (such as a Hive data warehouse) or complex, you may require more than 256 steps to process your data.
/// You can bypass the 256-step limitation in various ways, including using the SSH shell to connect to the master node and submitting queries
/// directly to the software running on the master node, such as Hive and Hadoop. For more information on how to do this, go to <a
/// href="http://docs.aws.amazon.com/ElasticMapReduce/latest/DeveloperGuide/AddMoreThan256Steps.html" >Add More than 256 Steps to a Job Flow</a>
/// in the <i>Amazon Elastic MapReduce Developer's Guide</i> .</para> <para>For long running job flows, we recommend that you periodically store
/// your results.</para>
/// </summary>
///
/// <param name="runJobFlowRequest">Container for the necessary parameters to execute the RunJobFlow service method on
/// AmazonElasticMapReduce.</param>
///
/// <returns>The response from the RunJobFlow service method, as returned by AmazonElasticMapReduce.</returns>
///
/// <exception cref="InternalServerErrorException"/>
public RunJobFlowResponse RunJobFlow(RunJobFlowRequest runJobFlowRequest)
{
IAsyncResult asyncResult = invokeRunJobFlow(runJobFlowRequest, null, null, true);
return EndRunJobFlow(asyncResult);
}
/// <summary>
/// <para> RunJobFlow creates and starts running a new job flow. The job
/// flow will run the steps specified. Once the job flow completes, the
/// cluster is stopped and the HDFS partition is lost. To prevent loss of
/// data, configure the last step of the job flow to store results in
/// Amazon S3. If the JobFlowInstancesDetail : KeepJobFlowAliveWhenNoSteps
/// parameter is set to <c>TRUE</c> , the job flow will transition to the
/// WAITING state rather than shutting down once the steps have completed.
/// </para> <para>A maximum of 256 steps are allowed in each job
/// flow.</para> <para>For long running job flows, we recommended that you
/// periodically store your results.</para>
/// </summary>
///
/// <param name="runJobFlowRequest">Container for the necessary parameters
/// to execute the RunJobFlow service method on
/// AmazonElasticMapReduce.</param>
///
/// <returns>The response from the RunJobFlow service method, as returned
/// by AmazonElasticMapReduce.</returns>
///
/// <exception cref="InternalServerErrorException"/>
public RunJobFlowResponse RunJobFlow(RunJobFlowRequest runJobFlowRequest)
{
IRequest<RunJobFlowRequest> request = new RunJobFlowRequestMarshaller().Marshall(runJobFlowRequest);
RunJobFlowResponse response = Invoke<RunJobFlowRequest, RunJobFlowResponse> (request, this.signer, RunJobFlowResponseUnmarshaller.GetInstance());
return response;
}
/// <summary>
/// <para> RunJobFlow creates and starts running a new job flow. The job flow will run the steps specified. Once the job flow completes, the
/// cluster is stopped and the HDFS partition is lost. To prevent loss of data, configure the last step of the job flow to store results in
/// Amazon S3. If the JobFlowInstancesConfig <c>KeepJobFlowAliveWhenNoSteps</c> parameter is set to <c>TRUE</c> , the job flow will transition
/// to the WAITING state rather than shutting down once the steps have completed. </para> <para>For additional protection, you can set the
/// JobFlowInstancesConfig <c>TerminationProtected</c> parameter to <c>TRUE</c> to lock the job flow and prevent it from being terminated by API
/// call, user intervention, or in the event of a job flow error.</para> <para>A maximum of 256 steps are allowed in each job flow.</para>
/// <para>If your job flow is long-running (such as a Hive data warehouse) or complex, you may require more than 256 steps to process your data.
/// You can bypass the 256-step limitation in various ways, including using the SSH shell to connect to the master node and submitting queries
/// directly to the software running on the master node, such as Hive and Hadoop. For more information on how to do this, go to <a href="http://docs.aws.amazon.com/ElasticMapReduce/latest/DeveloperGuide/AddMoreThan256Steps.html">Add More than 256 Steps to a Job Flow</a>
/// in the <i>Amazon Elastic MapReduce Developer's Guide</i> .</para> <para>For long running job flows, we recommend that you periodically store
/// your results.</para>
/// </summary>
///
/// <param name="runJobFlowRequest">Container for the necessary parameters to execute the RunJobFlow service method on
/// AmazonElasticMapReduce.</param>
///
/// <returns>The response from the RunJobFlow service method, as returned by AmazonElasticMapReduce.</returns>
///
/// <exception cref="T:Amazon.ElasticMapReduce.Model.InternalServerErrorException" />
/// <param name="cancellationToken">
/// A cancellation token that can be used by other objects or threads to receive notice of cancellation.
/// </param>
public async Task<RunJobFlowResponse> RunJobFlowAsync(RunJobFlowRequest runJobFlowRequest, CancellationToken cancellationToken = default(CancellationToken))
{
var marshaller = new RunJobFlowRequestMarshaller();
var unmarshaller = RunJobFlowResponseUnmarshaller.GetInstance();
var response = await Invoke<IRequest, RunJobFlowRequest, RunJobFlowResponse>(runJobFlowRequest, marshaller, unmarshaller, signer, cancellationToken)
.ConfigureAwait(continueOnCapturedContext: false);
return response;
}
public override ProvisionAddOnResult Provision(AddonProvisionRequest request)
{
var provisionResult = new ProvisionAddOnResult("") { IsSuccess = true };
AddonManifest manifest = request.Manifest;
string developerOptions = request.DeveloperOptions;
try
{
IAmazonElasticMapReduce client;
EMRDeveloperOptions devOptions;
var parseOptionsResult = ParseDevOptions(developerOptions, out devOptions);
if (!parseOptionsResult.IsSuccess)
{
provisionResult.EndUserMessage = parseOptionsResult.EndUserMessage;
return provisionResult;
}
var establishClientResult = EstablishClient(manifest, EMRDeveloperOptions.Parse(developerOptions), out client);
if (!establishClientResult.IsSuccess)
{
provisionResult.EndUserMessage = establishClientResult.EndUserMessage;
return provisionResult;
}
var stepFactory = new StepFactory();
StepConfig enabledebugging = null;
if (devOptions.EnableDebugging)
{
enabledebugging = new StepConfig
{
Name = "Enable debugging",
ActionOnFailure = "TERMINATE_JOB_FLOW",
HadoopJarStep = stepFactory.NewEnableDebuggingStep()
};
}
var installHive = new StepConfig
{
Name = "Install Hive",
ActionOnFailure = "TERMINATE_JOB_FLOW",
HadoopJarStep = stepFactory.NewInstallHiveStep()
};
var instanceConfig = new JobFlowInstancesConfig
{
Ec2KeyName = devOptions.Ec2KeyName,
HadoopVersion = "0.20",
InstanceCount = devOptions.InstanceCount,
// this is important. the EMR job flow must be kept alive for the application to see it during provisioning
KeepJobFlowAliveWhenNoSteps = true,
MasterInstanceType = devOptions.MasterInstanceType,
SlaveInstanceType = devOptions.SlaveInstanceType
};
var _request = new RunJobFlowRequest
{
Name = devOptions.JobFlowName,
Steps = { enabledebugging, installHive },
// revisit this one in ne
LogUri = "s3://myawsbucket",
Instances = instanceConfig
};
// if debugging is enabled, add to top of the list of steps.
if (devOptions.EnableDebugging)
{
_request.Steps.Insert(0, enabledebugging);
}
var result = client.RunJobFlow(_request);
// wait for JobFlowID to come back.
while (result.JobFlowId == null)
{
Thread.Sleep(1000);
}
provisionResult.IsSuccess = true;
provisionResult.ConnectionData = string.Format(result.JobFlowId);
}
catch (Exception e)
{
provisionResult.EndUserMessage = e.Message;
}
return provisionResult;
}
IAsyncResult invokeRunJobFlow(RunJobFlowRequest request, AsyncCallback callback, object state, bool synchronized)
{
var marshaller = new RunJobFlowRequestMarshaller();
var unmarshaller = RunJobFlowResponseUnmarshaller.Instance;
return Invoke(request, callback, state, synchronized, marshaller, unmarshaller, signer);
}
