/// <summary>
/// Submits a MapReduce streaming job to an HDINSIGHT cluster.
/// </summary>
/// <param name='operations'>
/// Reference to the
/// Microsoft.Azure.Management.HDInsight.Job.IJobOperations.
/// </param>
/// <param name='parameters'>
/// Required. MapReduce job parameters.
/// </param>
/// <returns>
/// The Create Job operation response.
/// </returns>
public static Task<JobSubmissionResponse> SubmitMapReduceStreamingJobAsync(this IJobOperations operations, MapReduceStreamingJobSubmissionParameters parameters)
{
return operations.SubmitMapReduceStreamingJobAsync(parameters, CancellationToken.None);
}
public void SubmitMapReduceStreamingJobWithFilesParam()
{
using (var context = UndoContext.Current)
{
context.Start();
var username = TestUtils.WinUserName;
var password = TestUtils.WinPassword;
var clustername = TestUtils.WinClusterName;
var credentials = new BasicAuthenticationCloudCredentials
{
Username = username,
Password = password
};
var client = TestUtils.GetHDInsightJobManagementClient(clustername, credentials);
var parameters = new MapReduceStreamingJobSubmissionParameters
{
Mapper = "cat.exe",
Reducer = "wc.exe",
Input = "/example/data/gutenberg/davinci.txt",
Output = "/example/data/gutenberg/wcount",
Files = new List<string> { "/example/apps/wc.exe", "/example/apps/cat.exe" }
};
var response = client.JobManagement.SubmitMapReduceStreamingJob(parameters);
Assert.NotNull(response);
Assert.Equal(response.StatusCode, HttpStatusCode.OK);
var jobId = response.JobSubmissionJsonResponse.Id;
Assert.Contains("job_", jobId, StringComparison.InvariantCulture);
var jobStatus = GetJobFinalStatus(client, jobId);
var storageAccess = new AzureStorageAccess(TestUtils.WinStorageAccountName, TestUtils.WinStorageAccountKey, TestUtils.WinDefaultContainer);
if (jobStatus.JobDetail.ExitValue == 0)
{
if (HttpMockServer.Mode == HttpRecorderMode.Record)
{
// Retrieve Job Output
var output = client.JobManagement.GetJobOutput(jobId, storageAccess);
string textOutput = Convert(output);
Assert.True(textOutput.Length > 0);
}
}
else
{
if (HttpMockServer.Mode == HttpRecorderMode.Record)
{
var output = client.JobManagement.GetJobErrorLogs(jobId, storageAccess);
string errorTextOutput = Convert(output);
Assert.NotNull(errorTextOutput);
}
Assert.True(false);
}
}
}
/// <summary>
/// Submits a MapReduce streaming job to an HDInsight cluster.
/// </summary>
/// <param name='operations'>
/// Reference to the
/// Microsoft.Azure.Management.HDInsight.Job.IJobOperations.
/// </param>
/// <param name='parameters'>
/// Required. MapReduce job parameters.
/// </param>
/// <returns>
/// The Create Job operation response.
/// </returns>
public static JobSubmissionResponse SubmitMapReduceStreamingJob(this IJobOperations operations, MapReduceStreamingJobSubmissionParameters parameters)
{
return operations.SubmitMapReduceStreamingJob(new JobSubmissionParameters { Content = parameters.GetJobPostRequestContent() });
}
/// <summary>
/// Submits a MapReduce streaming job to an HDINSIGHT cluster.
/// </summary>
/// <param name='operations'>
/// Reference to the
/// Microsoft.Azure.Management.HDInsight.Job.IJobOperations.
/// </param>
/// <param name='parameters'>
/// Required. MapReduce job parameters.
/// </param>
/// <returns>
/// The Create Job operation response.
/// </returns>
public static JobSubmissionResponse SubmitMapReduceStreamingJob(this IJobOperations operations, MapReduceStreamingJobSubmissionParameters parameters)
{
return Task.Factory.StartNew((object s) =>
{
return ((IJobOperations)s).SubmitMapReduceStreamingJobAsync(parameters);
}
, operations, CancellationToken.None, TaskCreationOptions.None, TaskScheduler.Default).Unwrap().GetAwaiter().GetResult();
}
public void SubmitMapReduceStreamingJobAndValidateOutput(MapReduceStreamingJobSubmissionParameters parameters, bool runAyncAPI = false, bool isWindowsCluster = false)
{
using (var context = UndoContext.Current)
{
context.Start();
var client = TestUtils.GetHDInsightJobManagementClient(isWindowsCluster);
var response = runAyncAPI ? client.JobManagement.SubmitMapReduceStreamingJobAsync(parameters).Result
: client.JobManagement.SubmitMapReduceStreamingJob(parameters);
Assert.NotNull(response);
Assert.Equal(response.StatusCode, HttpStatusCode.OK);
var jobId = response.JobSubmissionJsonResponse.Id;
Assert.Contains("job_", jobId, StringComparison.InvariantCulture);
var jobStatus = client.JobManagement.WaitForJobCompletion(jobId, TestUtils.JobWaitInterval, TestUtils.JobPollInterval);
var storageAccess = GetStorageAccessObject(isWindowsCluster);
if (jobStatus.JobDetail.ExitValue == 0)
{
if (HttpMockServer.GetCurrentMode() == HttpRecorderMode.Record)
{
// Retrieve Job Output
var output = client.JobManagement.GetJobOutput(jobId, storageAccess);
string textOutput = Convert(output);
Assert.True(textOutput.Length > 0);
}
}
else
{
if (HttpMockServer.GetCurrentMode() == HttpRecorderMode.Record)
{
var output = client.JobManagement.GetJobErrorLogs(jobId, storageAccess);
string errorTextOutput = Convert(output);
Assert.NotNull(errorTextOutput);
}
Assert.True(false);
}
}
}
public MapReduceStreamingJobSubmissionParameters GetMRStreamingJobSubmissionParameters(bool isWindowsCluster = false)
{
var parameters = new MapReduceStreamingJobSubmissionParameters
{
Mapper = isWindowsCluster ? "cat.exe" : "cat",
Reducer = isWindowsCluster ? "wc.exe" : "wc",
Input = "/example/data/gutenberg/davinci.txt",
Output = "/example/data/gutenberg/wcount/" + Guid.NewGuid()
};
if (isWindowsCluster)
{
parameters.Files = new List<string> { "/example/apps/wc.exe", "/example/apps/cat.exe" };
}
return parameters;
}
public void SubmitMapReduceStreamingJob()
{
using (var context = UndoContext.Current)
{
context.Start();
var username = TestUtils.UserName;
var password = TestUtils.Password;
var clustername = TestUtils.ClusterName;
var credentials = new BasicAuthenticationCloudCredentials
{
Username = username,
Password = password
};
var client = TestUtils.GetHDInsightJobManagementClient(clustername, credentials);
var parameters = new MapReduceStreamingJobSubmissionParameters
{
UserName = username,
Mapper = "cat.exe",
Reducer = "wc.exe",
Input = "/example/data/gutenberg/davinci.txt",
Output = "/example/data/gutenberg/wcout"
};
var response = client.JobManagement.SubmitMapReduceStreamingJob(parameters);
Assert.NotNull(response);
Assert.Equal(response.StatusCode, HttpStatusCode.OK);
Assert.Contains("job_", response.JobSubmissionJsonResponse.Id, StringComparison.InvariantCulture);
}
}
public virtual JobSubmissionResponse SubmitStreamingJob(
AzureHDInsightStreamingMapReduceJobDefinition streamingJobDef)
{
var streamingJobParams = new MapReduceStreamingJobSubmissionParameters
{
Input = streamingJobDef.Input,
Output = streamingJobDef.Output,
Mapper = streamingJobDef.Mapper,
Reducer = streamingJobDef.Reducer,
File = streamingJobDef.File,
Defines = ConvertDefinesToString(streamingJobDef.Defines),
CmdEnv = ConvertListToString(streamingJobDef.CommandEnvironment, "cmdenv"),
Arguments = ConvertListToString(streamingJobDef.Arguments, "arg"),
StatusDir = streamingJobDef.StatusFolder,
UserName = HdInsightJobManagementClient.Credentials.Username
};
return HdInsightJobManagementClient.JobManagement.SubmitMapReduceStreamingJob(streamingJobParams);
}
public virtual JobSubmissionResponse SubmitStreamingJob(
AzureHDInsightStreamingMapReduceJobDefinition streamingJobDef)
{
var streamingJobParams = new MapReduceStreamingJobSubmissionParameters
{
Input = streamingJobDef.Input,
Output = streamingJobDef.Output,
Mapper = streamingJobDef.Mapper,
Reducer = streamingJobDef.Reducer,
File = streamingJobDef.File,
Files = streamingJobDef.Files,
Defines = streamingJobDef.Defines,
CmdEnv = streamingJobDef.CommandEnvironment,
Arguments = streamingJobDef.Arguments,
StatusDir = streamingJobDef.StatusFolder
};
return HdInsightJobManagementClient.JobManagement.SubmitMapReduceStreamingJob(streamingJobParams);
}
