private async Task TestLaunchJob()
{
//ARRANGE
StartJobRunRequest Request = new StartJobRunRequest()
{
JobName = "CUR File ETL",
Timeout = 1440, // 24 Hours
Arguments = new Dictionary <string, string>()
{
{ "--table", "2018-10-01" },
{ "--database", "billingdata" },
{ "--destination_bucket", $"{User}-billing-formatted" }
}
};
IAmazonGlue GlueClient;
if (Creds != null)
{
GlueClient = new AmazonGlueClient(Creds);
}
else
{
GlueClient = new AmazonGlueClient();
}
// ACT
StartJobRunResponse Response = await GlueClient.StartJobRunAsync(Request);
// ASSERT
Assert.NotNull(Response);
Assert.Equal(HttpStatusCode.OK, Response.HttpStatusCode);
}
internal virtual StartJobRunResponse StartJobRun(StartJobRunRequest request)
{
var options = new InvokeOptions();
options.RequestMarshaller = StartJobRunRequestMarshaller.Instance;
options.ResponseUnmarshaller = StartJobRunResponseUnmarshaller.Instance;
return(Invoke <StartJobRunResponse>(request, options));
}
/// <summary>
/// Starts a job run. A job run is a unit of work, such as a Spark jar, PySpark script,
/// or SparkSQL query, that you submit to Amazon EMR on EKS.
/// </summary>
/// <param name="request">Container for the necessary parameters to execute the StartJobRun service method.</param>
/// <param name="cancellationToken">
/// A cancellation token that can be used by other objects or threads to receive notice of cancellation.
/// </param>
///
/// <returns>The response from the StartJobRun service method, as returned by EMRContainers.</returns>
/// <exception cref="Amazon.EMRContainers.Model.InternalServerException">
/// This is an internal server exception.
/// </exception>
/// <exception cref="Amazon.EMRContainers.Model.ResourceNotFoundException">
/// The specified resource was not found.
/// </exception>
/// <exception cref="Amazon.EMRContainers.Model.ValidationException">
/// There are invalid parameters in the client request.
/// </exception>
/// <seealso href="http://docs.aws.amazon.com/goto/WebAPI/emr-containers-2020-10-01/StartJobRun">REST API Reference for StartJobRun Operation</seealso>
public virtual Task <StartJobRunResponse> StartJobRunAsync(StartJobRunRequest request, System.Threading.CancellationToken cancellationToken = default(CancellationToken))
{
var options = new InvokeOptions();
options.RequestMarshaller = StartJobRunRequestMarshaller.Instance;
options.ResponseUnmarshaller = StartJobRunResponseUnmarshaller.Instance;
return(InvokeAsync <StartJobRunResponse>(request, options, cancellationToken));
}
/// <summary>
/// If provided, runs a Glue job after the files have been copied
/// </summary>
/// <param name="context"></param>
/// <returns></returns>
private static async Task RunGlueJob(string table, ILambdaContext context)
{
if (String.IsNullOrEmpty(table))
{
throw new ArgumentNullException("table");
}
if (String.IsNullOrEmpty(_GlueDatabaseName))
{
string Message = "The Glue database name was provided. Not running job.";
context.LogWarning(Message);
await SNSNotify(Message, context);
return;
}
if (String.IsNullOrEmpty(_GlueJobName))
{
string Message = "The Glue job name for the job was not provided as an environment variable. Not running job.";
context.LogWarning(Message);
await SNSNotify(Message, context);
return;
}
context.LogInfo($"Running glue job on table {table} in database {_GlueDatabaseName}.");
try
{
StartJobRunRequest Request = new StartJobRunRequest()
{
JobName = _GlueJobName,
Timeout = 1440, // 24 Hours
Arguments = new Dictionary <string, string>()
{
{ "--table", table },
{ "--database", _GlueDatabaseName }
}
};
if (!String.IsNullOrEmpty(_GlueDestinationBucket))
{
Request.Arguments.Add("--destination_bucket", _GlueDestinationBucket);
}
StartJobRunResponse Response = await _GlueClient.StartJobRunAsync(Request);
if (Response.HttpStatusCode != HttpStatusCode.OK)
{
string Message = $"Failed to start job with status code ${(int)Response.HttpStatusCode}";
context.LogError(Message);
await SNSNotify(Message, context);
}
else
{
context.LogInfo($"Successfully started job {Response.JobRunId}");
}
}
catch (Exception e)
{
string Message = "Failed to start Glue job.";
context.LogError(Message, e);
await SNSNotify(Message + $" {e.Message}", context);
}
}
