public void SubmitMapReduceStreamingJob()
{
using (var context = UndoContext.Current)
{
context.Start();
var username = TestUtils.UserName;
var password = TestUtils.Password;
var clustername = TestUtils.ClusterName;
var credentials = new BasicAuthenticationCloudCredentials
{
Username = username,
Password = password
};
var client = TestUtils.GetHDInsightJobManagementClient(clustername, credentials);
var parameters = new MapReduceStreamingJobSubmissionParameters
{
Mapper = "cat",
Reducer = "wc",
Input = "/example/data/gutenberg/davinci.txt",
Output = "/example/data/gutenberg/wcount"
};
var response = client.JobManagement.SubmitMapReduceStreamingJob(parameters);
Assert.NotNull(response);
Assert.Equal(response.StatusCode, HttpStatusCode.OK);
var jobId = response.JobSubmissionJsonResponse.Id;
Assert.Contains("job_", jobId, StringComparison.InvariantCulture);
var jobStatus = GetJobFinalStatus(client, jobId);
var storageAccess = GetStorageAccessObject();
if (jobStatus.JobDetail.ExitValue == 0)
{
if (HttpMockServer.Mode == HttpRecorderMode.Record)
{
// Retrieve Job Output
var output = client.JobManagement.GetJobOutput(jobId, storageAccess);
string textOutput = Convert(output);
Assert.True(textOutput.Length > 0);
}
}
else
{
if (HttpMockServer.Mode == HttpRecorderMode.Record)
{
var output = client.JobManagement.GetJobErrorLogs(jobId, storageAccess);
string errorTextOutput = Convert(output);
Assert.NotNull(errorTextOutput);
}
Assert.True(false);
}
}
}
public void SubmitMapReduceStreamingJob()
{
using (var context = UndoContext.Current)
{
context.Start();
var username = TestUtils.UserName;
var password = TestUtils.Password;
var clustername = TestUtils.ClusterName;
var credentials = new BasicAuthenticationCloudCredentials
{
Username = username,
Password = password
};
var client = TestUtils.GetHDInsightJobManagementClient(clustername, credentials);
var parameters = new MapReduceStreamingJobSubmissionParameters
{
UserName = username,
Mapper = "cat.exe",
Reducer = "wc.exe",
Input = "/example/data/gutenberg/davinci.txt",
Output = "/example/data/gutenberg/wcout"
};
var response = client.JobManagement.SubmitMapReduceStreamingJob(parameters);
Assert.NotNull(response);
Assert.Equal(response.StatusCode, HttpStatusCode.OK);
Assert.Contains("job_", response.JobSubmissionJsonResponse.Id, StringComparison.InvariantCulture);
}
}
private static void SubmitMRJob()
{
//var paras = new MapReduceStreamingJobSubmissionParameters
//{
// Files = new List<string>() { "/example/app/.exe", "/example/apps/wc.exe" },
// Mapper = "cat.exe",
// Reducer = "wc.exe",
// Input= "/example/data/gutenberg/davinci.txt",
// Output = "/example/data/StreamingOutput/wc.txt"
//};
var paras = new MapReduceStreamingJobSubmissionParameters
{
Files = new List <string>()
{
"/example/coreapp",
},
Mapper = "dotnet coreapp/NetCoreMapper.dll",
Reducer = "dotnet coreapp/NetCoreReducer.dll",
Input = "/example/data/gutenberg/davinci.txt",
Output = "/example/data/StreamingOutput/wc.txt"
};
Console.WriteLine("Submitting the MR job to the cluster...");
var jobResponse = _hdiJobManagementClient.JobManagement.SubmitMapReduceStreamingJob(paras);
var jobId = jobResponse.JobSubmissionJsonResponse.Id;
Console.WriteLine("Response status code is " + jobResponse.StatusCode);
Console.WriteLine("JobId is " + jobId);
Console.WriteLine("Waiting for the job completion ...");
// Wait for job completion
var jobDetail = _hdiJobManagementClient.JobManagement.GetJob(jobId).JobDetail;
while (!jobDetail.Status.JobComplete)
{
Thread.Sleep(1000);
jobDetail = _hdiJobManagementClient.JobManagement.GetJob(jobId).JobDetail;
}
// Get job output
var storageAccess = new AzureStorageAccess(DefaultStorageAccountName, DefaultStorageAccountKey,
DefaultStorageContainerName);
var output = (jobDetail.ExitValue == 0)
? _hdiJobManagementClient.JobManagement.GetJobOutput(jobId, storageAccess) // fetch stdout output in case of success
: _hdiJobManagementClient.JobManagement.GetJobErrorLogs(jobId, storageAccess); // fetch stderr output in case of failure
Console.WriteLine("Job output is: ");
using (var reader = new StreamReader(output, Encoding.UTF8))
{
string value = reader.ReadToEnd();
Console.WriteLine(value);
}
}
public MapReduceStreamingJobSubmissionParameters GetMRStreamingJobSubmissionParameters()
{
var parameters = new MapReduceStreamingJobSubmissionParameters
{
Mapper = "cat",
Reducer = "wc",
Input = "/example/data/gutenberg/davinci.txt",
Output = "/example/data/gutenberg/wcount/" + Guid.NewGuid()
};
return(parameters);
}
public void SubmitMapReduceStreamingJobAndValidateOutput(MapReduceStreamingJobSubmissionParameters parameters, bool runAyncAPI = false, bool isWindowsCluster = false)
{
using (var context = UndoContext.Current)
{
context.Start();
var client = TestUtils.GetHDInsightJobManagementClient(isWindowsCluster);
var response = runAyncAPI ? client.JobManagement.SubmitMapReduceStreamingJobAsync(parameters).Result
: client.JobManagement.SubmitMapReduceStreamingJob(parameters);
Assert.NotNull(response);
Assert.Equal(response.StatusCode, HttpStatusCode.OK);
var jobId = response.JobSubmissionJsonResponse.Id;
Assert.Contains("job_", jobId, StringComparison.InvariantCulture);
WaitForJobCompletion(client, jobId, TestUtils.JobPollInterval, TestUtils.JobWaitInterval);
var jobStatus = client.JobManagement.GetJob(jobId);
var storageAccess = GetStorageAccessObject(isWindowsCluster);
if (jobStatus.JobDetail.ExitValue == 0)
{
if (HttpMockServer.Mode == HttpRecorderMode.Record)
{
// Retrieve Job Output
var output = client.JobManagement.GetJobOutput(jobId, storageAccess);
string textOutput = Convert(output);
Assert.True(textOutput.Length > 0);
}
}
else
{
if (HttpMockServer.Mode == HttpRecorderMode.Record)
{
var output = client.JobManagement.GetJobErrorLogs(jobId, storageAccess);
string errorTextOutput = Convert(output);
Assert.NotNull(errorTextOutput);
}
Assert.True(false);
}
}
}
public virtual JobSubmissionResponse SubmitStreamingJob(
AzureHDInsightStreamingMapReduceJobDefinition streamingJobDef)
{
var streamingJobParams = new MapReduceStreamingJobSubmissionParameters
{
Input = streamingJobDef.Input,
Output = streamingJobDef.Output,
Mapper = streamingJobDef.Mapper,
Reducer = streamingJobDef.Reducer,
File = streamingJobDef.File,
Defines = ConvertDefinesToString(streamingJobDef.Defines),
CmdEnv = ConvertListToString(streamingJobDef.CommandEnvironment, "cmdenv"),
Arguments = ConvertListToString(streamingJobDef.Arguments, "arg"),
StatusDir = streamingJobDef.StatusFolder,
UserName = HdInsightJobManagementClient.Credentials.Username
};
return(HdInsightJobManagementClient.JobManagement.SubmitMapReduceStreamingJob(streamingJobParams));
}
public MapReduceStreamingJobSubmissionParameters GetMRStreamingJobSubmissionParameters(bool isWindowsCluster = false)
{
var parameters = new MapReduceStreamingJobSubmissionParameters
{
Mapper = isWindowsCluster ? "cat.exe" : "cat",
Reducer = isWindowsCluster ? "wc.exe" : "wc",
Input = "/example/data/gutenberg/davinci.txt",
Output = "/example/data/gutenberg/wcount/" + Guid.NewGuid()
};
if (isWindowsCluster)
{
parameters.Files = new List <string> {
"/example/apps/wc.exe", "/example/apps/cat.exe"
};
}
return(parameters);
}
public virtual JobSubmissionResponse SubmitStreamingJob(
AzureHDInsightStreamingMapReduceJobDefinition streamingJobDef)
{
var streamingJobParams = new MapReduceStreamingJobSubmissionParameters
{
Input = streamingJobDef.Input,
Output = streamingJobDef.Output,
Mapper = streamingJobDef.Mapper,
Reducer = streamingJobDef.Reducer,
File = streamingJobDef.File,
Files = streamingJobDef.Files,
Defines = streamingJobDef.Defines,
CmdEnv = streamingJobDef.CommandEnvironment,
Arguments = streamingJobDef.Arguments,
StatusDir = streamingJobDef.StatusFolder
};
return(HdInsightJobManagementClient.JobManagement.SubmitMapReduceStreamingJob(streamingJobParams));
}
private void SubmitMapReduceStreamingJobAndValidateOutput(MapReduceStreamingJobSubmissionParameters parameters, bool runAyncAPI = false)
{
using (var context = MockContext.Start(this.GetType()))
using (var client = this.CommonData.GetHDInsightJobClient(context))
{
var response = runAyncAPI ? client.Job.SubmitMapReduceStreamingJobAsync(parameters).Result
: client.Job.SubmitMapReduceStreamingJob(parameters);
Assert.NotNull(response);
var jobId = response.Id;
Assert.Contains("job_", jobId, StringComparison.InvariantCulture);
var jobStatus = client.Job.WaitForJobCompletion(jobId, this.CommonData.JobWaitInterval, this.CommonData.JobPollInterval);
var storageAccess = GetStorageAccessObject();
if (jobStatus.ExitValue == 0)
{
if (HttpMockServer.GetCurrentMode() == HttpRecorderMode.Record)
{
// Retrieve Job Output
var output = client.Job.GetJobOutput(jobId, storageAccess);
string textOutput = Convert(output);
Assert.True(textOutput.Length > 0);
}
}
else
{
if (HttpMockServer.GetCurrentMode() == HttpRecorderMode.Record)
{
var output = client.Job.GetJobErrorLogs(jobId, storageAccess);
string errorTextOutput = Convert(output);
Assert.NotNull(errorTextOutput);
}
Assert.True(false);
}
}
}
/// <summary>
/// Submits a MapReduce streaming job to an HDInsight cluster.
/// </summary>
/// <param name='parameters'>
/// Required. MapReduce job parameters.
/// </param>
/// <returns>
/// The Create Job operation response.
/// </returns>
public async Task <JobSubmissionResponse> SubmitMapReduceStreamingJobAsync(MapReduceStreamingJobSubmissionParameters parameters)
{
return(await SubmitMapReduceStreamingJobAsync(new JobSubmissionParameters { Content = parameters.GetJobPostRequestContent() }, CancellationToken.None));
}
/// <summary>
/// Submits a MapReduce streaming job to an HDInsight cluster.
/// </summary>
/// <param name='operations'>
/// Reference to the
/// Microsoft.Azure.HDInsight.Job.IJobOperations.
/// </param>
/// <param name='parameters'>
/// Required. MapReduce job parameters.
/// </param>
/// <returns>
/// The Create Job operation response.
/// </returns>
public static async Task <JobSubmissionJsonResponse> SubmitMapReduceStreamingJobAsync(this IJobOperations operations, MapReduceStreamingJobSubmissionParameters parameters, CancellationToken cancellationToken = default(CancellationToken))
{
using (var _result = await operations.SubmitMapReduceStreamingJobWithHttpMessagesAsync(parameters, null, cancellationToken).ConfigureAwait(false))
{
return(_result.Body);
}
}
/// <summary>
/// Submits a MapReduce streaming job to an HDInsight cluster.
/// </summary>
/// <param name='operations'>
/// Reference to the
/// Microsoft.Azure.HDInsight.Job.IJobOperations.
/// </param>
/// <param name='parameters'>
/// Required. MapReduce job parameters.
/// </param>
/// <returns>
/// The Create Job operation response.
/// </returns>
public static JobSubmissionJsonResponse SubmitMapReduceStreamingJob(this IJobOperations operations, MapReduceStreamingJobSubmissionParameters parameters)
{
return(operations.SubmitMapReduceStreamingJobAsync(parameters).GetAwaiter().GetResult());
}
/// <summary>
/// Submits a MapReduce streaming job to an HDInsight cluster.
/// </summary>
/// <param name='parameters'>
/// Required. MapReduce job parameters.
/// </param>
/// <returns>
/// The Create Job operation response.
/// </returns>
public async Task <AzureOperationResponse <JobSubmissionJsonResponse> > SubmitMapReduceStreamingJobWithHttpMessagesAsync(MapReduceStreamingJobSubmissionParameters parameters, Dictionary <string, List <string> > customHeaders = null, CancellationToken cancellationToken = default(CancellationToken))
{
using (var requestContents = new MemoryStream(Encoding.UTF8.GetBytes(parameters.GetJobPostRequestContent())))
{
return(await SubmitMapReduceStreamingJobWithHttpMessagesAsync(requestContents, customHeaders, cancellationToken).ConfigureAwait(false));
}
}
/// <summary>
/// Submits a MapReduce streaming job to an HDINSIGHT cluster.
/// </summary>
/// <param name='operations'>
/// Reference to the
/// Microsoft.Azure.Management.HDInsight.Job.IJobOperations.
/// </param>
/// <param name='parameters'>
/// Required. MapReduce job parameters.
/// </param>
/// <returns>
/// The Create Job operation response.
/// </returns>
public static Task <JobSubmissionResponse> SubmitMapReduceStreamingJobAsync(this IJobOperations operations, MapReduceStreamingJobSubmissionParameters parameters)
{
return(operations.SubmitMapReduceStreamingJobAsync(parameters, CancellationToken.None));
}
/// <summary>
/// Submits a MapReduce streaming job to an HDINSIGHT cluster.
/// </summary>
/// <param name='operations'>
/// Reference to the
/// Microsoft.Azure.Management.HDInsight.Job.IJobOperations.
/// </param>
/// <param name='parameters'>
/// Required. MapReduce job parameters.
/// </param>
/// <returns>
/// The Create Job operation response.
/// </returns>
public static JobSubmissionResponse SubmitMapReduceStreamingJob(this IJobOperations operations, MapReduceStreamingJobSubmissionParameters parameters)
{
return(Task.Factory.StartNew((object s) =>
{
return ((IJobOperations)s).SubmitMapReduceStreamingJobAsync(parameters);
}
, operations, CancellationToken.None, TaskCreationOptions.None, TaskScheduler.Default).Unwrap().GetAwaiter().GetResult());
}
/// <summary>
/// Submits a MapReduce streaming job to an HDInsight cluster.
/// </summary>
/// <param name='operations'>
/// Reference to the
/// Microsoft.Azure.Management.HDInsight.Job.IJobOperations.
/// </param>
/// <param name='parameters'>
/// Required. MapReduce job parameters.
/// </param>
/// <returns>
/// The Create Job operation response.
/// </returns>
public static JobSubmissionResponse SubmitMapReduceStreamingJob(this IJobOperations operations, MapReduceStreamingJobSubmissionParameters parameters)
{
return(operations.SubmitMapReduceStreamingJob(new JobSubmissionParameters {
Content = parameters.GetJobPostRequestContent()
}));
}
