private async Task TestLaunchJob() { //ARRANGE StartJobRunRequest Request = new StartJobRunRequest() { JobName = "CUR File ETL", Timeout = 1440, // 24 Hours Arguments = new Dictionary <string, string>() { { "--table", "2018-10-01" }, { "--database", "billingdata" }, { "--destination_bucket", $"{User}-billing-formatted" } } }; IAmazonGlue GlueClient; if (Creds != null) { GlueClient = new AmazonGlueClient(Creds); } else { GlueClient = new AmazonGlueClient(); } // ACT StartJobRunResponse Response = await GlueClient.StartJobRunAsync(Request); // ASSERT Assert.NotNull(Response); Assert.Equal(HttpStatusCode.OK, Response.HttpStatusCode); }
internal virtual StartJobRunResponse StartJobRun(StartJobRunRequest request) { var options = new InvokeOptions(); options.RequestMarshaller = StartJobRunRequestMarshaller.Instance; options.ResponseUnmarshaller = StartJobRunResponseUnmarshaller.Instance; return(Invoke <StartJobRunResponse>(request, options)); }
/// <summary> /// Starts a job run. A job run is a unit of work, such as a Spark jar, PySpark script, /// or SparkSQL query, that you submit to Amazon EMR on EKS. /// </summary> /// <param name="request">Container for the necessary parameters to execute the StartJobRun service method.</param> /// <param name="cancellationToken"> /// A cancellation token that can be used by other objects or threads to receive notice of cancellation. /// </param> /// /// <returns>The response from the StartJobRun service method, as returned by EMRContainers.</returns> /// <exception cref="Amazon.EMRContainers.Model.InternalServerException"> /// This is an internal server exception. /// </exception> /// <exception cref="Amazon.EMRContainers.Model.ResourceNotFoundException"> /// The specified resource was not found. /// </exception> /// <exception cref="Amazon.EMRContainers.Model.ValidationException"> /// There are invalid parameters in the client request. /// </exception> /// <seealso href="http://docs.aws.amazon.com/goto/WebAPI/emr-containers-2020-10-01/StartJobRun">REST API Reference for StartJobRun Operation</seealso> public virtual Task <StartJobRunResponse> StartJobRunAsync(StartJobRunRequest request, System.Threading.CancellationToken cancellationToken = default(CancellationToken)) { var options = new InvokeOptions(); options.RequestMarshaller = StartJobRunRequestMarshaller.Instance; options.ResponseUnmarshaller = StartJobRunResponseUnmarshaller.Instance; return(InvokeAsync <StartJobRunResponse>(request, options, cancellationToken)); }
/// <summary> /// If provided, runs a Glue job after the files have been copied /// </summary> /// <param name="context"></param> /// <returns></returns> private static async Task RunGlueJob(string table, ILambdaContext context) { if (String.IsNullOrEmpty(table)) { throw new ArgumentNullException("table"); } if (String.IsNullOrEmpty(_GlueDatabaseName)) { string Message = "The Glue database name was provided. Not running job."; context.LogWarning(Message); await SNSNotify(Message, context); return; } if (String.IsNullOrEmpty(_GlueJobName)) { string Message = "The Glue job name for the job was not provided as an environment variable. Not running job."; context.LogWarning(Message); await SNSNotify(Message, context); return; } context.LogInfo($"Running glue job on table {table} in database {_GlueDatabaseName}."); try { StartJobRunRequest Request = new StartJobRunRequest() { JobName = _GlueJobName, Timeout = 1440, // 24 Hours Arguments = new Dictionary <string, string>() { { "--table", table }, { "--database", _GlueDatabaseName } } }; if (!String.IsNullOrEmpty(_GlueDestinationBucket)) { Request.Arguments.Add("--destination_bucket", _GlueDestinationBucket); } StartJobRunResponse Response = await _GlueClient.StartJobRunAsync(Request); if (Response.HttpStatusCode != HttpStatusCode.OK) { string Message = $"Failed to start job with status code ${(int)Response.HttpStatusCode}"; context.LogError(Message); await SNSNotify(Message, context); } else { context.LogInfo($"Successfully started job {Response.JobRunId}"); } } catch (Exception e) { string Message = "Failed to start Glue job."; context.LogError(Message, e); await SNSNotify(Message + $" {e.Message}", context); } }