public virtual JobSubmissionResponse SubmitMRJob(AzureHDInsightMapReduceJobDefinition mapredJobDef)
{
var mapredJobParams = new MapReduceJobSubmissionParameters
{
Arguments = mapredJobDef.Arguments,
Defines = mapredJobDef.Defines,
Files = mapredJobDef.Files,
JarClass = mapredJobDef.ClassName,
LibJars = mapredJobDef.LibJars,
JarFile = mapredJobDef.JarFile,
StatusDir = mapredJobDef.StatusFolder
};
return(HdInsightJobManagementClient.JobManagement.SubmitMapReduceJob(mapredJobParams));
}
public void KillMapReduceStreamingJob()
{
using (var context = UndoContext.Current)
{
context.Start();
var username = TestUtils.UserName;
var password = TestUtils.Password;
var clustername = TestUtils.ClusterName;
var credentials = new BasicAuthenticationCloudCredentials
{
Username = username,
Password = password
};
var client = TestUtils.GetHDInsightJobManagementClient(clustername, credentials);
var defines = new Dictionary <string, string>
{
{ "mapreduce.map.maxattempts", "10" },
{ "mapreduce.reduce.maxattempts", "10" },
{ "mapreduce.task.timeout", "60000" }
};
var args = new List <string> {
"10", "1000"
};
var parameters = new MapReduceJobSubmissionParameters
{
UserName = username,
JarFile = "/example/jars/hadoop-mapreduce-examples.jar",
JarClass = "pi",
Defines = ConvertDefinesToString(defines),
Arguments = ConvertArgsToString(args)
};
var jobid = client.JobManagement.SubmitMapReduceJob(parameters).JobSubmissionJsonResponse.Id;
var response = client.JobManagement.GetJob(jobid);
Assert.NotNull(response);
Assert.Equal(response.StatusCode, HttpStatusCode.OK);
var job = client.JobManagement.KillJob(jobid);
Assert.NotNull(job);
Assert.Equal(job.JobDetail.Status.State, "KILLED");
}
}
public virtual JobSubmissionResponse SubmitMRJob(AzureHDInsightMapReduceJobDefinition mapredJobDef)
{
var mapredJobParams = new MapReduceJobSubmissionParameters
{
Arguments = ConvertListToString(mapredJobDef.Arguments, "arg"),
Defines = ConvertDefinesToString(mapredJobDef.Defines),
Files = ConvertListToString(mapredJobDef.Files, "file"),
JarClass = mapredJobDef.ClassName,
LibJars = ConvertListToString(mapredJobDef.LibJars, "jar"),
JarFile = mapredJobDef.JarFile,
StatusDir = mapredJobDef.StatusFolder,
UserName = HdInsightJobManagementClient.Credentials.Username
};
return(HdInsightJobManagementClient.JobManagement.SubmitMapReduceJob(mapredJobParams));
}
private MapReduceJobSubmissionParameters GetMapReduceJobParameters()
{
var defines = new Dictionary <string, string>
{
{ "mapreduce.map.maxattempts", "10" },
{ "mapreduce.reduce.maxattempts", "10" },
{ "mapreduce.task.timeout", "60000" }
};
var args = new List <string> {
"10", "1000"
};
var parameters = new MapReduceJobSubmissionParameters
{
JarFile = "/example/jars/hadoop-mapreduce-examples.jar",
JarClass = "pi",
Defines = defines,
Arguments = args
};
return(parameters);
}
/// <summary>
/// Submits a MapReduce job to an HDInsight cluster.
/// </summary>
/// <param name='parameters'>
/// Required. MapReduce job parameters.
/// </param>
/// <returns>
/// The Create Job operation response.
/// </returns>
public async Task <JobSubmissionResponse> SubmitMapReduceJobAsync(MapReduceJobSubmissionParameters parameters)
{
return(await SubmitMapReduceJobAsync(new JobSubmissionParameters { Content = parameters.GetJobPostRequestContent() }, CancellationToken.None));
}
/// <summary>
/// Submits a MapReduce job to an HDInsight cluster.
/// </summary>
/// <param name='operations'>
/// Reference to the
/// Microsoft.Azure.HDInsight.Job.IJobOperations.
/// </param>
/// <param name='parameters'>
/// Required. MapReduce job parameters.
/// </param>
/// <returns>
/// The Create Job operation response.
/// </returns>
public static async Task <JobSubmissionJsonResponse> SubmitMapReduceJobAsync(this IJobOperations operations, MapReduceJobSubmissionParameters parameters, CancellationToken cancellationToken = default(CancellationToken))
{
using (var _result = await operations.SubmitMapReduceJobWithHttpMessagesAsync(parameters, null, cancellationToken).ConfigureAwait(false))
{
return(_result.Body);
}
}
/// <summary>
/// Submits a MapReduce job to an HDInsight cluster.
/// </summary>
/// <param name='operations'>
/// Reference to the
/// Microsoft.Azure.HDInsight.Job.IJobOperations.
/// </param>
/// <param name='parameters'>
/// Required. MapReduce job parameters.
/// </param>
/// <returns>
/// The Create Job operation response.
/// </returns>
public static JobSubmissionJsonResponse SubmitMapReduceJob(this IJobOperations operations, MapReduceJobSubmissionParameters parameters)
{
return(operations.SubmitMapReduceJobAsync(parameters).GetAwaiter().GetResult());
}
/// <summary>
/// Submits a MapReduce job to an HDInsight cluster.
/// </summary>
/// <param name='parameters'>
/// Required. MapReduce job parameters.
/// </param>
/// <returns>
/// The Create Job operation response.
/// </returns>
public async Task <AzureOperationResponse <JobSubmissionJsonResponse> > SubmitMapReduceJobWithHttpMessagesAsync(MapReduceJobSubmissionParameters parameters, Dictionary <string, List <string> > customHeaders = null, CancellationToken cancellationToken = default(CancellationToken))
{
using (var requestContents = new MemoryStream(Encoding.UTF8.GetBytes(parameters.GetJobPostRequestContent())))
{
return(await SubmitMapReduceJobWithHttpMessagesAsync(requestContents, customHeaders, cancellationToken).ConfigureAwait(false));
}
}
/// <summary>
/// Submits a MapReduce job to an HDINSIGHT cluster.
/// </summary>
/// <param name='operations'>
/// Reference to the
/// Microsoft.Azure.Management.HDInsight.Job.IJobOperations.
/// </param>
/// <param name='parameters'>
/// Required. MapReduce job parameters.
/// </param>
/// <returns>
/// The Create Job operation response.
/// </returns>
public static Task <JobSubmissionResponse> SubmitMapReduceJobAsync(this IJobOperations operations, MapReduceJobSubmissionParameters parameters)
{
return(operations.SubmitMapReduceJobAsync(parameters, CancellationToken.None));
}
/// <summary>
/// Submits a MapReduce job to an HDINSIGHT cluster.
/// </summary>
/// <param name='operations'>
/// Reference to the
/// Microsoft.Azure.Management.HDInsight.Job.IJobOperations.
/// </param>
/// <param name='parameters'>
/// Required. MapReduce job parameters.
/// </param>
/// <returns>
/// The Create Job operation response.
/// </returns>
public static JobSubmissionResponse SubmitMapReduceJob(this IJobOperations operations, MapReduceJobSubmissionParameters parameters)
{
return(Task.Factory.StartNew((object s) =>
{
return ((IJobOperations)s).SubmitMapReduceJobAsync(parameters);
}
, operations, CancellationToken.None, TaskCreationOptions.None, TaskScheduler.Default).Unwrap().GetAwaiter().GetResult());
}
public void SubmitMapReduceJob()
{
using (var context = UndoContext.Current)
{
context.Start();
var username = TestUtils.UserName;
var password = TestUtils.Password;
var clustername = TestUtils.ClusterName;
var credentials = new BasicAuthenticationCloudCredentials
{
Username = username,
Password = password
};
var client = TestUtils.GetHDInsightJobManagementClient(clustername, credentials);
var defines = new Dictionary <string, string>
{
{ "mapreduce.map.maxattempts", "10" },
{ "mapreduce.reduce.maxattempts", "10" },
{ "mapreduce.task.timeout", "60000" }
};
var args = new List <string> {
"10", "1000"
};
var parameters = new MapReduceJobSubmissionParameters
{
JarFile = "/example/jars/hadoop-mapreduce-examples.jar",
JarClass = "pi",
Defines = defines,
Arguments = args
};
var response = client.JobManagement.SubmitMapReduceJob(parameters);
Assert.NotNull(response);
Assert.Equal(response.StatusCode, HttpStatusCode.OK);
var jobId = response.JobSubmissionJsonResponse.Id;
Assert.Contains("job_", jobId, StringComparison.InvariantCulture);
var jobStatus = GetJobFinalStatus(client, jobId);
var storageAccess = GetStorageAccessObject();
if (jobStatus.JobDetail.ExitValue == 0)
{
if (HttpMockServer.Mode == HttpRecorderMode.Record)
{
// Retrieve Job Output
var output = client.JobManagement.GetJobOutput(jobId, storageAccess);
string textOutput = Convert(output);
Assert.True(textOutput.Contains("Estimated value of Pi is 3.14"));
}
}
else
{
if (HttpMockServer.Mode == HttpRecorderMode.Record)
{
var output = client.JobManagement.GetJobErrorLogs(jobId, storageAccess);
string errorTextOutput = Convert(output);
Assert.NotNull(errorTextOutput);
}
Assert.True(false);
}
}
}
/// <summary>
/// Submits a MapReduce job to an HDInsight cluster.
/// </summary>
/// <param name='operations'>
/// Reference to the
/// Microsoft.Azure.Management.HDInsight.Job.IJobOperations.
/// </param>
/// <param name='parameters'>
/// Required. MapReduce job parameters.
/// </param>
/// <returns>
/// The Create Job operation response.
/// </returns>
public static JobSubmissionResponse SubmitMapReduceJob(this IJobOperations operations, MapReduceJobSubmissionParameters parameters)
{
return(operations.SubmitMapReduceJob(new JobSubmissionParameters {
Content = parameters.GetJobPostRequestContent()
}));
}
