public void ICanCallThe_New_HDInsightStreamingMapReduceJobDefinitionCmdlet()
{
var streamingMapReduceJobDefinition = new StreamingMapReduceJobCreateParameters
{
JobName = "pi estimation jobDetails",
Input = TestConstants.WabsProtocolSchemeName + "input",
Output = TestConstants.WabsProtocolSchemeName + "input",
Mapper = TestConstants.WabsProtocolSchemeName + "combiner",
Reducer = TestConstants.WabsProtocolSchemeName + "combiner",
StatusFolder = TestConstants.WabsProtocolSchemeName + "someotherlocation"
};
using (IRunspace runspace = this.GetPowerShellRunspace())
{
IPipelineResult results =
runspace.NewPipeline()
.AddCommand(CmdletConstants.NewAzureHDInsightStreamingMapReduceJobDefinition)
.WithParameter(CmdletConstants.JobName, streamingMapReduceJobDefinition.JobName)
.WithParameter(CmdletConstants.Input, streamingMapReduceJobDefinition.Input)
.WithParameter(CmdletConstants.Output, streamingMapReduceJobDefinition.Output)
.WithParameter(CmdletConstants.Mapper, streamingMapReduceJobDefinition.Mapper)
.WithParameter(CmdletConstants.Reducer, streamingMapReduceJobDefinition.Reducer)
.WithParameter(CmdletConstants.StatusFolder, streamingMapReduceJobDefinition.StatusFolder)
.Invoke();
Assert.AreEqual(1, results.Results.Count);
AzureHDInsightStreamingMapReduceJobDefinition streamingMapReduceJobFromPowershell =
results.Results.ToEnumerable<AzureHDInsightStreamingMapReduceJobDefinition>().First();
AssertJobDefinitionsEqual(streamingMapReduceJobDefinition, streamingMapReduceJobFromPowershell);
}
}
public virtual void NewStreamingMapReduceJob_StartJob_GetJob()
{
var streamingMapReduceJobDefinition = new StreamingMapReduceJobCreateParameters
{
JobName = "environment variable jobDetails",
Mapper = "environmentvariables.exe",
Input = "/example/apps/environmentvariables.exe",
Output = Guid.NewGuid().ToString()
};
var files = new List <string> {
streamingMapReduceJobDefinition.Mapper
};
using (IRunspace runspace = this.GetPowerShellRunspace())
{
IPipelineResult results =
runspace.NewPipeline()
.AddCommand(CmdletConstants.NewAzureHDInsightStreamingMapReduceJobDefinition)
.WithParameter(CmdletConstants.JobName, streamingMapReduceJobDefinition.JobName)
.WithParameter(CmdletConstants.Mapper, streamingMapReduceJobDefinition.Mapper)
.WithParameter(CmdletConstants.Input, streamingMapReduceJobDefinition.Input)
.WithParameter(CmdletConstants.Output, streamingMapReduceJobDefinition.Output)
.WithParameter(CmdletConstants.Files, files)
.Invoke();
Assert.AreEqual(1, results.Results.Count);
AzureHDInsightStreamingMapReduceJobDefinition streamingMapReduceJobFromPowershell =
results.Results.ToEnumerable <AzureHDInsightStreamingMapReduceJobDefinition>().First();
RunJobAndGetWithId(runspace, streamingMapReduceJobFromPowershell);
}
}
/// <summary>
/// Creates the payload for a Streaming MapReduce request.
/// </summary>
/// <param name="userName">
/// The user name.
/// </param>
/// <param name="details">
/// The details.
/// </param>
/// <returns>
/// A string that represents the payload for the request.
/// </returns>
public string SerializeStreamingMapReduceRequest(string userName, StreamingMapReduceJobCreateParameters details)
{
details.ArgumentNotNull("details");
var values = new List <KeyValuePair <string, string> >();
values.Add(new KeyValuePair <string, string>(WebHCatConstants.Input, details.Input));
values.Add(new KeyValuePair <string, string>(WebHCatConstants.Output, details.Output));
values.Add(new KeyValuePair <string, string>(WebHCatConstants.Mapper, details.Mapper));
if (!details.Combiner.IsNullOrEmpty())
{
values.Add(new KeyValuePair <string, string>(WebHCatConstants.Combiner, details.Combiner));
}
if (details.Reducer.IsNullOrEmpty())
{
values.Add(new KeyValuePair <string, string>(WebHCatConstants.Reducer, NoneReducer));
}
else
{
values.Add(new KeyValuePair <string, string>(WebHCatConstants.Reducer, details.Reducer));
}
values.AddRange(this.SerializeJobRequest(userName, details, details.JobName, details.Arguments, details.Defines));
values.AddRange(this.BuildList(WebHCatConstants.CmdEnv, details.CommandEnvironment));
var retval = this.ConvertItems(values.Where(kvp => kvp.Value != null));
return(retval);
}
public void CanSerializeValidStreamingMapReduceJobRequest_CmdEnv()
{
var streamingMapReduceJob = new StreamingMapReduceJobCreateParameters()
{
Input = "asv://input",
Output = "asv://output",
Mapper = "asv://mapper",
Reducer = "asv://reducer"
};
streamingMapReduceJob.CommandEnvironment.Add("Name1=Value1");
streamingMapReduceJob.CommandEnvironment.Add("Name2=Value2");
var payloadConverter = new PayloadConverterBase();
var payload = payloadConverter.SerializeStreamingMapReduceRequest("hadoop", streamingMapReduceJob);
Assert.IsTrue(payload.Contains(string.Format("{0}={1}", WebHCatConstants.Input, Uri.EscapeDataString(streamingMapReduceJob.Input))));
Assert.IsTrue(payload.Contains(string.Format("{0}={1}", WebHCatConstants.Output, Uri.EscapeDataString(streamingMapReduceJob.Output))));
Assert.IsTrue(payload.Contains(string.Format("{0}={1}", WebHCatConstants.Mapper, Uri.EscapeDataString(streamingMapReduceJob.Mapper))));
Assert.IsTrue(payload.Contains(string.Format("{0}={1}", WebHCatConstants.Reducer, Uri.EscapeDataString(streamingMapReduceJob.Reducer))));
foreach (var cmdEnvArgument in streamingMapReduceJob.CommandEnvironment)
{
Assert.IsTrue(payload.Contains(string.Format("{0}={1}", WebHCatConstants.CmdEnv, Uri.EscapeDataString(cmdEnvArgument))));
}
}
public void CanSerializeValidStreamingMapReduceJobRequest_WithFiles()
{
var streamingMapReduceJob = new StreamingMapReduceJobCreateParameters()
{
Input = Constants.WabsProtocolSchemeName + "input",
Output = Constants.WabsProtocolSchemeName + "output",
Mapper = Constants.WabsProtocolSchemeName + "mapper",
Reducer = Constants.WabsProtocolSchemeName + "reducer"
};
var resourceFile1 = "asv://container@hostname/myfile1";
var resourceFile2 = "asv://container@hostname/myfile2";
var payloadConverter = new PayloadConverterBase();
streamingMapReduceJob.Files.Add(resourceFile1);
streamingMapReduceJob.Files.Add(resourceFile2);
var payload = payloadConverter.SerializeStreamingMapReduceRequest("hadoop", streamingMapReduceJob);
Assert.IsTrue(payload.Contains(string.Format("{0}={1}", WebHCatConstants.Input, Uri.EscapeDataString(streamingMapReduceJob.Input))));
Assert.IsTrue(payload.Contains(string.Format("{0}={1}", WebHCatConstants.Output, Uri.EscapeDataString(streamingMapReduceJob.Output))));
Assert.IsTrue(payload.Contains(string.Format("{0}={1}", WebHCatConstants.Mapper, Uri.EscapeDataString(streamingMapReduceJob.Mapper))));
Assert.IsTrue(payload.Contains(string.Format("{0}={1}", WebHCatConstants.Reducer, Uri.EscapeDataString(streamingMapReduceJob.Reducer))));
Assert.IsTrue(payload.Contains(Uri.EscapeDataString(resourceFile1)));
Assert.IsTrue(payload.Contains(Uri.EscapeDataString(resourceFile2)));
}
public void CanSerializeValidStreamingMapReduceJobRequest_Defines()
{
var streamingMapReduceJob = new StreamingMapReduceJobCreateParameters()
{
Input = "asv://input",
Output = "asv://output",
Mapper = "asv://mapper",
Reducer = "asv://reducer",
};
streamingMapReduceJob.Defines.Add(new KeyValuePair <string, string>("definekey1", "definevalue1"));
streamingMapReduceJob.Defines.Add(new KeyValuePair <string, string>("definekey2", "definevalue2"));
streamingMapReduceJob.Defines.Add(new KeyValuePair <string, string>("definekey3", "definevalue3"));
var payloadConverter = new PayloadConverterBase();
var payload = payloadConverter.SerializeStreamingMapReduceRequest("hadoop", streamingMapReduceJob);
Assert.IsTrue(payload.Contains(string.Format("{0}={1}", WebHCatConstants.Input, Uri.EscapeDataString(streamingMapReduceJob.Input))));
Assert.IsTrue(payload.Contains(string.Format("{0}={1}", WebHCatConstants.Output, Uri.EscapeDataString(streamingMapReduceJob.Output))));
Assert.IsTrue(payload.Contains(string.Format("{0}={1}", WebHCatConstants.Mapper, Uri.EscapeDataString(streamingMapReduceJob.Mapper))));
Assert.IsTrue(payload.Contains(string.Format("{0}={1}", WebHCatConstants.Reducer, Uri.EscapeDataString(streamingMapReduceJob.Reducer))));
foreach (var define in streamingMapReduceJob.Defines)
{
Assert.IsTrue(payload.Contains(Uri.EscapeDataString(string.Format("{0}={1}", define.Key, define.Value))));
}
}
public void CanCreateNewStreamingMapReduceDefinition_WithArguments()
{
StreamingMapReduceJobCreateParameters streamingMapReduceJobDefinition = this.GetStreamingMapReduceJob();
streamingMapReduceJobDefinition.Arguments.Add("16");
streamingMapReduceJobDefinition.Arguments.Add("10000");
INewAzureHDInsightStreamingJobDefinitionCommand newStreamingMapReduceJobDefinitionCommand =
ServiceLocator.Instance.Locate <IAzureHDInsightCommandFactory>().CreateNewStreamingMapReduceDefinition();
newStreamingMapReduceJobDefinitionCommand.JobName = streamingMapReduceJobDefinition.JobName;
newStreamingMapReduceJobDefinitionCommand.JobName = streamingMapReduceJobDefinition.JobName;
newStreamingMapReduceJobDefinitionCommand.InputPath = streamingMapReduceJobDefinition.Input;
newStreamingMapReduceJobDefinitionCommand.OutputPath = streamingMapReduceJobDefinition.Output;
newStreamingMapReduceJobDefinitionCommand.Mapper = streamingMapReduceJobDefinition.Mapper;
newStreamingMapReduceJobDefinitionCommand.Reducer = streamingMapReduceJobDefinition.Reducer;
newStreamingMapReduceJobDefinitionCommand.StatusFolder = streamingMapReduceJobDefinition.StatusFolder;
newStreamingMapReduceJobDefinitionCommand.Arguments = streamingMapReduceJobDefinition.Arguments.ToArray();
newStreamingMapReduceJobDefinitionCommand.EndProcessing();
AzureHDInsightStreamingMapReduceJobDefinition streamingMapReduceJobFromCommand =
newStreamingMapReduceJobDefinitionCommand.Output.ElementAt(0);
NewStreamingMapReduceJobCmdLetTests.AssertJobDefinitionsEqual(streamingMapReduceJobDefinition, streamingMapReduceJobFromCommand);
foreach (string argument in streamingMapReduceJobDefinition.Arguments)
{
Assert.IsTrue(
streamingMapReduceJobFromCommand.Arguments.Any(arg => string.Equals(argument, arg)),
"Unable to find argument '{0}' in value returned from command",
argument);
}
}
public void CanCreateNewStreamingJob_StartJob()
{
var streamingMapReduceJobDefinition = new StreamingMapReduceJobCreateParameters
{
JobName = "pi estimation jobDetails",
Input = TestConstants.WabsProtocolSchemeName + "container@hostname/input",
Output = TestConstants.WabsProtocolSchemeName + "container@hostname/input",
Mapper = TestConstants.WabsProtocolSchemeName + "container@hostname/combiner",
Reducer = TestConstants.WabsProtocolSchemeName + "container@hostname/combiner",
StatusFolder = TestConstants.WabsProtocolSchemeName + "container@hostname/someotherlocation"
};
INewAzureHDInsightStreamingJobDefinitionCommand newStreamingMapReduceJobDefinitionCommand =
ServiceLocator.Instance.Locate <IAzureHDInsightCommandFactory>().CreateNewStreamingMapReduceDefinition();
newStreamingMapReduceJobDefinitionCommand.JobName = streamingMapReduceJobDefinition.JobName;
newStreamingMapReduceJobDefinitionCommand.JobName = streamingMapReduceJobDefinition.JobName;
newStreamingMapReduceJobDefinitionCommand.InputPath = streamingMapReduceJobDefinition.Input;
newStreamingMapReduceJobDefinitionCommand.OutputPath = streamingMapReduceJobDefinition.Output;
newStreamingMapReduceJobDefinitionCommand.Mapper = streamingMapReduceJobDefinition.Mapper;
newStreamingMapReduceJobDefinitionCommand.Reducer = streamingMapReduceJobDefinition.Reducer;
newStreamingMapReduceJobDefinitionCommand.StatusFolder = streamingMapReduceJobDefinition.StatusFolder;
newStreamingMapReduceJobDefinitionCommand.EndProcessing();
AzureHDInsightStreamingMapReduceJobDefinition streamingJobFromCommand = newStreamingMapReduceJobDefinitionCommand.Output.ElementAt(0);
TestJobStart(streamingJobFromCommand);
}
/// <summary>
/// Creates the payload for a Streaming MapReduce request.
/// </summary>
/// <param name="userName">
/// The user name.
/// </param>
/// <param name="details">
/// The details.
/// </param>
/// <returns>
/// A string that represents the payload for the request.
/// </returns>
public string SerializeStreamingMapReduceRequest(string userName, StreamingMapReduceJobCreateParameters details)
{
details.ArgumentNotNull("details");
var values = new List<KeyValuePair<string, string>>();
values.Add(new KeyValuePair<string, string>(WebHCatConstants.Input, details.Input));
values.Add(new KeyValuePair<string, string>(WebHCatConstants.Output, details.Output));
values.Add(new KeyValuePair<string, string>(WebHCatConstants.Mapper, details.Mapper));
if (!details.Combiner.IsNullOrEmpty())
{
values.Add(new KeyValuePair<string, string>(WebHCatConstants.Combiner, details.Combiner));
}
if (details.Reducer.IsNullOrEmpty())
{
values.Add(new KeyValuePair<string, string>(WebHCatConstants.Reducer, NoneReducer));
}
else
{
values.Add(new KeyValuePair<string, string>(WebHCatConstants.Reducer, details.Reducer));
}
values.AddRange(this.SerializeJobRequest(userName, details, details.JobName, details.Arguments, details.Defines));
values.AddRange(this.BuildList(WebHCatConstants.CmdEnv, details.CommandEnvironment));
var retval = this.ConvertItems(values.Where(kvp => kvp.Value != null));
return retval;
}
public virtual void ICanCallThe_NewStreamingJob_Then_Start_HDInsightJobsCmdlet()
{
var streamingMapReduceJobDefinition = new StreamingMapReduceJobCreateParameters
{
JobName = "pi estimation jobDetails",
Input = TestConstants.WabsProtocolSchemeName + "input",
Output = TestConstants.WabsProtocolSchemeName + "input",
Mapper = TestConstants.WabsProtocolSchemeName + "combiner",
Reducer = TestConstants.WabsProtocolSchemeName + "combiner",
StatusFolder = TestConstants.WabsProtocolSchemeName + "someotherlocation"
};
using (IRunspace runspace = this.GetPowerShellRunspace())
{
IPipelineResult results =
runspace.NewPipeline()
.AddCommand(CmdletConstants.NewAzureHDInsightStreamingMapReduceJobDefinition)
.WithParameter(CmdletConstants.JobName, streamingMapReduceJobDefinition.JobName)
.WithParameter(CmdletConstants.Input, streamingMapReduceJobDefinition.Input)
.WithParameter(CmdletConstants.Output, streamingMapReduceJobDefinition.Output)
.WithParameter(CmdletConstants.Mapper, streamingMapReduceJobDefinition.Mapper)
.WithParameter(CmdletConstants.Reducer, streamingMapReduceJobDefinition.Reducer)
.WithParameter(CmdletConstants.StatusFolder, streamingMapReduceJobDefinition.StatusFolder)
.Invoke();
Assert.AreEqual(1, results.Results.Count);
AzureHDInsightStreamingMapReduceJobDefinition streamingJobFromPowershell =
results.Results.ToEnumerable <AzureHDInsightStreamingMapReduceJobDefinition>().First();
RunJobInPowershell(runspace, streamingJobFromPowershell);
}
}
public Task <JobCreationResults> SubmitStreamingJob(StreamingMapReduceJobCreateParameters streamingMapReduceJobCreateParameters)
{
if (streamingMapReduceJobCreateParameters == null)
{
throw new ArgumentNullException("streamingMapReduceJobCreateParameters");
}
return(Task.FromResult(this.CreateJobSuccessResult(streamingMapReduceJobCreateParameters, streamingMapReduceJobCreateParameters.JobName)));
}
internal static void AssertJobDefinitionsEqual(
StreamingMapReduceJobCreateParameters streamingMapReduceJobDefinition,
AzureHDInsightStreamingMapReduceJobDefinition streamingMapReduceJobFromPowershell)
{
Assert.AreEqual(streamingMapReduceJobDefinition.JobName, streamingMapReduceJobFromPowershell.JobName);
Assert.AreEqual(streamingMapReduceJobDefinition.Input, streamingMapReduceJobFromPowershell.Input);
Assert.AreEqual(streamingMapReduceJobDefinition.Output, streamingMapReduceJobFromPowershell.Output);
Assert.AreEqual(streamingMapReduceJobDefinition.Mapper, streamingMapReduceJobFromPowershell.Mapper);
Assert.AreEqual(streamingMapReduceJobDefinition.Reducer, streamingMapReduceJobFromPowershell.Reducer);
Assert.AreEqual(streamingMapReduceJobDefinition.StatusFolder, streamingMapReduceJobFromPowershell.StatusFolder);
}
/// <inheritdoc />
public async Task <JobCreationResults> SubmitStreamingJob(StreamingMapReduceJobCreateParameters streamingJobCreateParameters)
{
//NEIN: Any code modification here should add unit tests for this class
var converter = new PayloadConverter();
var payload = converter.SerializeStreamingMapReduceRequest(this.credentials.UserName, streamingJobCreateParameters);
var client = ServiceLocator.Instance.Locate <IHadoopRemoteJobSubmissionRestClientFactory>().Create(this.credentials, this.context, this.ignoreSslErrors, this.GetUserAgentString());
var result = await client.SubmitStreamingMapReduceJob(payload);
return(new JobCreationResults()
{
JobId = converter.DeserializeJobSubmissionResponse(result.Content)
});
}
/// <inheritdoc/>
public async Task <JobCreationResults> CreateStreamingJobAsync(StreamingMapReduceJobCreateParameters streamingMapReduceJobCreateParameters)
{
var pocoClient = this.GetPocoClient();
try
{
return(await pocoClient.SubmitStreamingJob(streamingMapReduceJobCreateParameters));
}
catch (UnauthorizedAccessException)
{
pocoClient = this.GetPocoClient(true);
}
return(await pocoClient.SubmitStreamingJob(streamingMapReduceJobCreateParameters));
}
public async Task <JobCreationResults> SubmitStreamingJob(StreamingMapReduceJobCreateParameters pigJobCreateParameters)
{
var relative = new Uri(
HadoopRemoteRestConstants.Sqoop + "?" +
HadoopRemoteRestConstants.UserName + "=" +
_connectionConfig.UserName.EscapeDataString(),
UriKind.Relative);
var requestContent = _converter.SerializeStreamingMapReduceRequest(_connectionConfig.UserName, pigJobCreateParameters);
var result = await MakeAsyncJobSubmissionRequest(relative, requestContent);
return(new JobCreationResults {
JobId = _converter.DeserializeJobSubmissionResponse(result)
});
}
public void CanSerializeValidStreamingMapReduceJobRequest_WithNoReducer()
{
var streamingMapReduceJob = new StreamingMapReduceJobCreateParameters()
{
Input = Constants.WabsProtocolSchemeName + "input",
Output = Constants.WabsProtocolSchemeName + "output",
Mapper = Constants.WabsProtocolSchemeName + "mapper"
};
var payloadConverter = new PayloadConverterBase();
var payload = payloadConverter.SerializeStreamingMapReduceRequest("hadoop", streamingMapReduceJob);
Assert.IsTrue(payload.Contains(string.Format("{0}={1}", WebHCatConstants.Input, Uri.EscapeDataString(streamingMapReduceJob.Input))));
Assert.IsTrue(payload.Contains(string.Format("{0}={1}", WebHCatConstants.Output, Uri.EscapeDataString(streamingMapReduceJob.Output))));
Assert.IsTrue(payload.Contains(string.Format("{0}={1}", WebHCatConstants.Mapper, Uri.EscapeDataString(streamingMapReduceJob.Mapper))));
}
public ClusterJob Submit(XDocument config, Uri errorLocation)
{
// upload config xml doc to blob
// launch single task map only job to submit xonfig xml to yarn launcher
// get job output which will be the application id
Uri userDfs = this.DfsClient.Combine(this.baseUri, "user", Environment.UserName, "staging");
Guid jobGuid = Guid.NewGuid();
StreamingMapReduceJobCreateParameters jobParams = new StreamingMapReduceJobCreateParameters();
jobParams.Defines.Add("mapreduce.job.reduces", "0");
jobParams.Defines.Add("mapreduce.job.maps", "1");
jobParams.JobName = "YarnMapLauncher";
jobParams.Mapper = "Microsoft.Research.Peloponnese.YarnMapLauncher.exe";
Uri outputDir = this.DfsClient.Combine(userDfs, jobGuid.ToString());
//dfsClient removes the leading slash, but hadoop assumes job output is relative to user working directory
jobParams.Output = this.dfsClient.GetClusterInternalUri(outputDir).AbsoluteUri;
using (MemoryStream upload = new MemoryStream())
{
XmlWriterSettings settings = new XmlWriterSettings();
settings.Encoding = new UTF8Encoding(false);
using (XmlWriter xw = XmlWriter.Create(upload, settings))
{
config.WriteTo(xw);
}
XElement elem = ConfigHelpers.MakeDfsResourceFromBuffer(jobGuid + ".xml", upload.ToArray(), this.DfsClient, userDfs);
jobParams.Input = elem.Value;
}
Uri peloponneseDfs = this.DfsClient.Combine(this.baseUri, "staging", "peloponnese");
string[] files = { jobParams.Mapper, "Microsoft.Research.Peloponnese.YarnLauncher.jar" };
foreach (string file in files)
{
XElement elem = ConfigHelpers.MakeDfsResourceFromFile(Path.Combine(peloponneseHome, file),
this.DfsClient, peloponneseDfs);
// hash mark in resources allows a target name to be specified for the symlink
jobParams.Files.Add(string.Format("{0}#{1}", elem.Value, file));
}
var launchJob = JobClient.CreateStreamingJob(jobParams);
return(new AzureYarnJob(this, errorLocation, outputDir, launchJob));
}
public void ICanCallThe_New_HDInsightStreamingMapReduceJobDefinitionCmdlet_WithParameters()
{
var streamingMapReduceJobDefinition = new StreamingMapReduceJobCreateParameters
{
JobName = "pi estimation jobDetails",
Input = TestConstants.WabsProtocolSchemeName + "input",
Output = TestConstants.WabsProtocolSchemeName + "input",
Mapper = TestConstants.WabsProtocolSchemeName + "combiner",
Reducer = TestConstants.WabsProtocolSchemeName + "combiner",
StatusFolder = TestConstants.WabsProtocolSchemeName + "someotherlocation"
};
streamingMapReduceJobDefinition.Defines.Add("map.input.tasks", "1000");
streamingMapReduceJobDefinition.Defines.Add("map.input.reducers", "1000");
using (IRunspace runspace = this.GetPowerShellRunspace())
{
IPipelineResult results =
runspace.NewPipeline()
.AddCommand(CmdletConstants.NewAzureHDInsightStreamingMapReduceJobDefinition)
.WithParameter(CmdletConstants.JobName, streamingMapReduceJobDefinition.JobName)
.WithParameter(CmdletConstants.Input, streamingMapReduceJobDefinition.Input)
.WithParameter(CmdletConstants.Output, streamingMapReduceJobDefinition.Output)
.WithParameter(CmdletConstants.Mapper, streamingMapReduceJobDefinition.Mapper)
.WithParameter(CmdletConstants.Reducer, streamingMapReduceJobDefinition.Reducer)
.WithParameter(CmdletConstants.StatusFolder, streamingMapReduceJobDefinition.StatusFolder)
.WithParameter(CmdletConstants.Parameters, streamingMapReduceJobDefinition.Defines)
.Invoke();
Assert.AreEqual(1, results.Results.Count);
AzureHDInsightStreamingMapReduceJobDefinition streamingMapReduceJobFromPowershell =
results.Results.ToEnumerable <AzureHDInsightStreamingMapReduceJobDefinition>().First();
AssertJobDefinitionsEqual(streamingMapReduceJobDefinition, streamingMapReduceJobFromPowershell);
foreach (var parameter in streamingMapReduceJobDefinition.Defines)
{
Assert.IsTrue(
streamingMapReduceJobFromPowershell.Defines.Any(
arg => string.Equals(parameter.Key, arg.Key) && string.Equals(parameter.Value, arg.Value)),
"Unable to find parameter '{0}' in value returned from powershell",
parameter.Key);
}
}
}
public void CanSerializeValidStreamingMapReduceJobRequest()
{
var streamingMapReduceJob = new StreamingMapReduceJobCreateParameters()
{
Input = "asv://input",
Output = "asv://output",
Mapper = "asv://mapper",
Reducer = "asv://reducer",
Combiner = "asv://combiner"
};
var payloadConverter = new PayloadConverterBase();
var payload = payloadConverter.SerializeStreamingMapReduceRequest("hadoop", streamingMapReduceJob);
Assert.IsTrue(payload.Contains(string.Format("{0}={1}", WebHCatConstants.Input, Uri.EscapeDataString(streamingMapReduceJob.Input))));
Assert.IsTrue(payload.Contains(string.Format("{0}={1}", WebHCatConstants.Output, Uri.EscapeDataString(streamingMapReduceJob.Output))));
Assert.IsTrue(payload.Contains(string.Format("{0}={1}", WebHCatConstants.Mapper, Uri.EscapeDataString(streamingMapReduceJob.Mapper))));
Assert.IsTrue(payload.Contains(string.Format("{0}={1}", WebHCatConstants.Reducer, Uri.EscapeDataString(streamingMapReduceJob.Reducer))));
Assert.IsTrue(payload.Contains(string.Format("{0}={1}", WebHCatConstants.Combiner, Uri.EscapeDataString(streamingMapReduceJob.Combiner))));
}
internal StreamingMapReduceJobCreateParameters ToStreamingMapReduceJobCreateParameters()
{
var streamingJobDefinition = new StreamingMapReduceJobCreateParameters
{
JobName = this.JobName,
Input = this.Input,
Output = this.Output,
Reducer = this.Reducer,
Combiner = this.Combiner,
Mapper = this.Mapper,
StatusFolder = this.StatusFolder
};
streamingJobDefinition.CommandEnvironment.AddRange(this.CommandEnvironment);
streamingJobDefinition.Arguments.AddRange(this.Arguments);
streamingJobDefinition.Defines.AddRange(this.Defines);
streamingJobDefinition.Files.AddRange(this.Files);
return streamingJobDefinition;
}
internal StreamingMapReduceJobCreateParameters ToStreamingMapReduceJobCreateParameters()
{
var streamingJobDefinition = new StreamingMapReduceJobCreateParameters
{
JobName = this.JobName,
Input = this.Input,
Output = this.Output,
Reducer = this.Reducer,
Combiner = this.Combiner,
Mapper = this.Mapper,
StatusFolder = this.StatusFolder
};
streamingJobDefinition.CommandEnvironment.AddRange(this.CommandEnvironment);
streamingJobDefinition.Arguments.AddRange(this.Arguments);
streamingJobDefinition.Defines.AddRange(this.Defines);
streamingJobDefinition.Files.AddRange(this.Files);
return(streamingJobDefinition);
}
public void ICanCallThe_New_HDInsightStreamingMapReduceJobDefinitionCmdlet_WithArguments()
{
var streamingMapReduceJobDefinition = new StreamingMapReduceJobCreateParameters
{
JobName = "pi estimation jobDetails",
Input = TestConstants.WabsProtocolSchemeName + "input",
Output = TestConstants.WabsProtocolSchemeName + "input",
Mapper = TestConstants.WabsProtocolSchemeName + "combiner",
Reducer = TestConstants.WabsProtocolSchemeName + "combiner",
StatusFolder = TestConstants.WabsProtocolSchemeName + "someotherlocation"
};
streamingMapReduceJobDefinition.Arguments.Add("16");
streamingMapReduceJobDefinition.Arguments.Add("10000");
using (IRunspace runspace = this.GetPowerShellRunspace())
{
IPipelineResult results =
runspace.NewPipeline()
.AddCommand(CmdletConstants.NewAzureHDInsightStreamingMapReduceJobDefinition)
.WithParameter(CmdletConstants.JobName, streamingMapReduceJobDefinition.JobName)
.WithParameter(CmdletConstants.Input, streamingMapReduceJobDefinition.Input)
.WithParameter(CmdletConstants.Output, streamingMapReduceJobDefinition.Output)
.WithParameter(CmdletConstants.Mapper, streamingMapReduceJobDefinition.Mapper)
.WithParameter(CmdletConstants.Reducer, streamingMapReduceJobDefinition.Reducer)
.WithParameter(CmdletConstants.StatusFolder, streamingMapReduceJobDefinition.StatusFolder)
.WithParameter(CmdletConstants.Arguments, streamingMapReduceJobDefinition.Arguments)
.Invoke();
Assert.AreEqual(1, results.Results.Count);
AzureHDInsightStreamingMapReduceJobDefinition streamingMapReduceJobFromPowershell =
results.Results.ToEnumerable<AzureHDInsightStreamingMapReduceJobDefinition>().First();
AssertJobDefinitionsEqual(streamingMapReduceJobDefinition, streamingMapReduceJobFromPowershell);
foreach (string argument in streamingMapReduceJobDefinition.Arguments)
{
Assert.IsTrue(
streamingMapReduceJobFromPowershell.Arguments.Any(arg => string.Equals(argument, arg)),
"Unable to find argument '{0}' in value returned from powershell",
argument);
}
}
}
public void CanCreateNewStreamingMapReduceDefinition()
{
StreamingMapReduceJobCreateParameters streamingMapReduceJobDefinition = this.GetStreamingMapReduceJob();
INewAzureHDInsightStreamingJobDefinitionCommand newStreamingMapReduceJobDefinitionCommand =
ServiceLocator.Instance.Locate <IAzureHDInsightCommandFactory>().CreateNewStreamingMapReduceDefinition();
newStreamingMapReduceJobDefinitionCommand.JobName = streamingMapReduceJobDefinition.JobName;
newStreamingMapReduceJobDefinitionCommand.JobName = streamingMapReduceJobDefinition.JobName;
newStreamingMapReduceJobDefinitionCommand.InputPath = streamingMapReduceJobDefinition.Input;
newStreamingMapReduceJobDefinitionCommand.OutputPath = streamingMapReduceJobDefinition.Output;
newStreamingMapReduceJobDefinitionCommand.Mapper = streamingMapReduceJobDefinition.Mapper;
newStreamingMapReduceJobDefinitionCommand.Reducer = streamingMapReduceJobDefinition.Reducer;
newStreamingMapReduceJobDefinitionCommand.StatusFolder = streamingMapReduceJobDefinition.StatusFolder;
newStreamingMapReduceJobDefinitionCommand.EndProcessing();
AzureHDInsightStreamingMapReduceJobDefinition streamingMapReduceJobFromCommand =
newStreamingMapReduceJobDefinitionCommand.Output.ElementAt(0);
NewStreamingMapReduceJobCmdLetTests.AssertJobDefinitionsEqual(streamingMapReduceJobDefinition, streamingMapReduceJobFromCommand);
}
public void CanCreateNewStreamingMapReduceDefinition_WithParameters()
{
StreamingMapReduceJobCreateParameters streamingMapReduceJobDefinition = this.GetStreamingMapReduceJob();
streamingMapReduceJobDefinition.Defines.Add("map.input.tasks", "1000");
streamingMapReduceJobDefinition.Defines.Add("map.input.reducers", "1000");
INewAzureHDInsightStreamingJobDefinitionCommand newStreamingMapReduceJobDefinitionCommand =
ServiceLocator.Instance.Locate <IAzureHDInsightCommandFactory>().CreateNewStreamingMapReduceDefinition();
newStreamingMapReduceJobDefinitionCommand.JobName = streamingMapReduceJobDefinition.JobName;
newStreamingMapReduceJobDefinitionCommand.JobName = streamingMapReduceJobDefinition.JobName;
newStreamingMapReduceJobDefinitionCommand.InputPath = streamingMapReduceJobDefinition.Input;
newStreamingMapReduceJobDefinitionCommand.OutputPath = streamingMapReduceJobDefinition.Output;
newStreamingMapReduceJobDefinitionCommand.Mapper = streamingMapReduceJobDefinition.Mapper;
newStreamingMapReduceJobDefinitionCommand.Reducer = streamingMapReduceJobDefinition.Reducer;
newStreamingMapReduceJobDefinitionCommand.StatusFolder = streamingMapReduceJobDefinition.StatusFolder;
newStreamingMapReduceJobDefinitionCommand.Defines = new Hashtable();
foreach (var define in streamingMapReduceJobDefinition.Defines)
{
newStreamingMapReduceJobDefinitionCommand.Defines.Add(define.Key, define.Value);
}
newStreamingMapReduceJobDefinitionCommand.EndProcessing();
AzureHDInsightStreamingMapReduceJobDefinition streamingMapReduceJobFromCommand =
newStreamingMapReduceJobDefinitionCommand.Output.ElementAt(0);
NewStreamingMapReduceJobCmdLetTests.AssertJobDefinitionsEqual(streamingMapReduceJobDefinition, streamingMapReduceJobFromCommand);
foreach (var parameter in streamingMapReduceJobDefinition.Defines)
{
Assert.IsTrue(
streamingMapReduceJobFromCommand.Defines.Any(
arg => string.Equals(parameter.Key, arg.Key) && string.Equals(parameter.Value, arg.Value)),
"Unable to find parameter '{0}' in value returned from command",
parameter.Key);
}
}
private static async Task <JobCreationResults> CreateStreamingJob(
AzureHDInsightStreamingMapReduceJobDefinition azureStreamingJobDefinition, IJobSubmissionClient client)
{
var streamingJobDefinition = new StreamingMapReduceJobCreateParameters
{
JobName = azureStreamingJobDefinition.JobName,
Input = azureStreamingJobDefinition.Input,
Output = azureStreamingJobDefinition.Output,
Reducer = azureStreamingJobDefinition.Reducer,
Combiner = azureStreamingJobDefinition.Combiner,
Mapper = azureStreamingJobDefinition.Mapper
};
streamingJobDefinition.StatusFolder = azureStreamingJobDefinition.StatusFolder;
streamingJobDefinition.CommandEnvironment.AddRange(azureStreamingJobDefinition.CommandEnvironment);
streamingJobDefinition.Arguments.AddRange(azureStreamingJobDefinition.Arguments);
streamingJobDefinition.Defines.AddRange(azureStreamingJobDefinition.Defines);
streamingJobDefinition.Files.AddRange(azureStreamingJobDefinition.Files);
var jobCreationResults = await client.CreateStreamingJobAsync(streamingJobDefinition);
return(jobCreationResults);
}
public virtual void PipeliningStreamingMapReduceJobExecution()
{
var streamingMapReduceJobDefinition = new StreamingMapReduceJobCreateParameters
{
JobName = "pi estimation job",
Mapper = "environmentvariables.exe",
Input = "/example/apps/environmentvariables.exe",
Output = Guid.NewGuid().ToString()
};
var files = new List <string> {
streamingMapReduceJobDefinition.Mapper
};
using (IRunspace runspace = this.GetPowerShellRunspace())
{
ClusterDetails testCluster = GetHttpAccessEnabledCluster();
IPipelineResult results =
runspace.NewPipeline()
.AddCommand(CmdletConstants.NewAzureHDInsightStreamingMapReduceJobDefinition)
.WithParameter(CmdletConstants.JobName, streamingMapReduceJobDefinition.JobName)
.WithParameter(CmdletConstants.Mapper, streamingMapReduceJobDefinition.Mapper)
.WithParameter(CmdletConstants.Input, streamingMapReduceJobDefinition.Input)
.WithParameter(CmdletConstants.Output, streamingMapReduceJobDefinition.Output)
.WithParameter(CmdletConstants.Files, files)
.AddCommand(CmdletConstants.StartAzureHDInsightJob)
.WithParameter(CmdletConstants.Cluster, testCluster.Name)
.AddCommand(CmdletConstants.WaitAzureHDInsightJob)
.WithParameter(CmdletConstants.WaitTimeoutInSeconds, 10)
.AddCommand(CmdletConstants.GetAzureHDInsightJobOutput)
.WithParameter(CmdletConstants.Cluster, testCluster.Name)
.Invoke();
Assert.IsNotNull(results.Results.ToEnumerable <string>());
Assert.IsTrue(results.Results.ToEnumerable <string>().Any(str => str == "3.142"));
}
}
/// <inheritdoc />
public async Task <JobCreationResults> SubmitStreamingJob(StreamingMapReduceJobCreateParameters pigJobCreateParameters)
{
var remoteClient = ServiceLocator.Instance.Locate <IRemoteHadoopJobSubmissionPocoClientFactory>().Create(this.remoteCreds, this.context, this.ignoreSslErrors, this.GetUserAgentString());
return(await remoteClient.SubmitStreamingJob(pigJobCreateParameters));
}
public Task <JobCreationResults> SubmitStreamingJob(StreamingMapReduceJobCreateParameters pigJobCreateParameters)
{
throw new System.NotImplementedException();
}
internal static void AssertJobDefinitionsEqual(
StreamingMapReduceJobCreateParameters streamingMapReduceJobDefinition,
AzureHDInsightStreamingMapReduceJobDefinition streamingMapReduceJobFromPowershell)
{
Assert.AreEqual(streamingMapReduceJobDefinition.JobName, streamingMapReduceJobFromPowershell.JobName);
Assert.AreEqual(streamingMapReduceJobDefinition.Input, streamingMapReduceJobFromPowershell.Input);
Assert.AreEqual(streamingMapReduceJobDefinition.Output, streamingMapReduceJobFromPowershell.Output);
Assert.AreEqual(streamingMapReduceJobDefinition.Mapper, streamingMapReduceJobFromPowershell.Mapper);
Assert.AreEqual(streamingMapReduceJobDefinition.Reducer, streamingMapReduceJobFromPowershell.Reducer);
Assert.AreEqual(streamingMapReduceJobDefinition.StatusFolder, streamingMapReduceJobFromPowershell.StatusFolder);
}
public JobCreationResults CreateStreamingJob(StreamingMapReduceJobCreateParameters streamingMapReduceJobCreateParameters)
{
return this.CreateStreamingJobAsync(streamingMapReduceJobCreateParameters).WaitForResult();
}
public Task<JobCreationResults> CreateStreamingJobAsync(StreamingMapReduceJobCreateParameters streamingMapReduceJobCreateParameters)
{
if (streamingMapReduceJobCreateParameters == null)
{
throw new ArgumentNullException("streamingMapReduceJobCreateParameters");
}
return
TaskEx2.FromResult(this.CreateJobSuccessResult(streamingMapReduceJobCreateParameters, streamingMapReduceJobCreateParameters.JobName));
}
/// <inheritdoc />
public async Task<JobCreationResults> SubmitStreamingJob(StreamingMapReduceJobCreateParameters streamingJobCreateParameters)
{
//NEIN: Any code modification here should add unit tests for this class
var converter = new PayloadConverter();
var payload = converter.SerializeStreamingMapReduceRequest(this.credentials.UserName, streamingJobCreateParameters);
var client = ServiceLocator.Instance.Locate<IHadoopRemoteJobSubmissionRestClientFactory>().Create(this.credentials, this.context, this.ignoreSslErrors, this.GetUserAgentString());
var result = await client.SubmitStreamingMapReduceJob(payload);
return new JobCreationResults() { JobId = converter.DeserializeJobSubmissionResponse(result.Content) };
}
public async Task<JobCreationResults> SubmitStreamingJob(StreamingMapReduceJobCreateParameters pigJobCreateParameters)
{
var relative = new Uri(
HadoopRemoteRestConstants.Sqoop + "?" +
HadoopRemoteRestConstants.UserName + "=" +
_connectionConfig.UserName.EscapeDataString(),
UriKind.Relative);
var requestContent = _converter.SerializeStreamingMapReduceRequest(_connectionConfig.UserName, pigJobCreateParameters);
var result = await MakeAsyncJobSubmissionRequest(relative, requestContent);
return new JobCreationResults { JobId = _converter.DeserializeJobSubmissionResponse(result) };
}
public void CanCreateNewStreamingJob_StartJob()
{
var streamingMapReduceJobDefinition = new StreamingMapReduceJobCreateParameters
{
JobName = "pi estimation jobDetails",
Input = TestConstants.WabsProtocolSchemeName + "container@hostname/input",
Output = TestConstants.WabsProtocolSchemeName + "container@hostname/input",
Mapper = TestConstants.WabsProtocolSchemeName + "container@hostname/combiner",
Reducer = TestConstants.WabsProtocolSchemeName + "container@hostname/combiner",
StatusFolder = TestConstants.WabsProtocolSchemeName + "container@hostname/someotherlocation"
};
INewAzureHDInsightStreamingJobDefinitionCommand newStreamingMapReduceJobDefinitionCommand =
ServiceLocator.Instance.Locate<IAzureHDInsightCommandFactory>().CreateNewStreamingMapReduceDefinition();
newStreamingMapReduceJobDefinitionCommand.JobName = streamingMapReduceJobDefinition.JobName;
newStreamingMapReduceJobDefinitionCommand.JobName = streamingMapReduceJobDefinition.JobName;
newStreamingMapReduceJobDefinitionCommand.InputPath = streamingMapReduceJobDefinition.Input;
newStreamingMapReduceJobDefinitionCommand.OutputPath = streamingMapReduceJobDefinition.Output;
newStreamingMapReduceJobDefinitionCommand.Mapper = streamingMapReduceJobDefinition.Mapper;
newStreamingMapReduceJobDefinitionCommand.Reducer = streamingMapReduceJobDefinition.Reducer;
newStreamingMapReduceJobDefinitionCommand.StatusFolder = streamingMapReduceJobDefinition.StatusFolder;
newStreamingMapReduceJobDefinitionCommand.EndProcessing();
AzureHDInsightStreamingMapReduceJobDefinition streamingJobFromCommand = newStreamingMapReduceJobDefinitionCommand.Output.ElementAt(0);
TestJobStart(streamingJobFromCommand);
}
public JobCreationResults CreateStreamingJob(StreamingMapReduceJobCreateParameters streamingMapReduceJobCreateParameters)
{
return(this.CreateStreamingJobAsync(streamingMapReduceJobCreateParameters).WaitForResult());
}
