public void ICanCallThe_New_HDInsightStreamingMapReduceJobDefinitionCmdlet()
{
var streamingMapReduceJobDefinition = new StreamingMapReduceJobCreateParameters
{
JobName = "pi estimation jobDetails",
Input = TestConstants.WabsProtocolSchemeName + "input",
Output = TestConstants.WabsProtocolSchemeName + "input",
Mapper = TestConstants.WabsProtocolSchemeName + "combiner",
Reducer = TestConstants.WabsProtocolSchemeName + "combiner",
StatusFolder = TestConstants.WabsProtocolSchemeName + "someotherlocation"
};
using (IRunspace runspace = this.GetPowerShellRunspace())
{
IPipelineResult results =
runspace.NewPipeline()
.AddCommand(CmdletConstants.NewAzureHDInsightStreamingMapReduceJobDefinition)
.WithParameter(CmdletConstants.JobName, streamingMapReduceJobDefinition.JobName)
.WithParameter(CmdletConstants.Input, streamingMapReduceJobDefinition.Input)
.WithParameter(CmdletConstants.Output, streamingMapReduceJobDefinition.Output)
.WithParameter(CmdletConstants.Mapper, streamingMapReduceJobDefinition.Mapper)
.WithParameter(CmdletConstants.Reducer, streamingMapReduceJobDefinition.Reducer)
.WithParameter(CmdletConstants.StatusFolder, streamingMapReduceJobDefinition.StatusFolder)
.Invoke();
Assert.AreEqual(1, results.Results.Count);
AzureHDInsightStreamingMapReduceJobDefinition streamingMapReduceJobFromPowershell =
results.Results.ToEnumerable<AzureHDInsightStreamingMapReduceJobDefinition>().First();
AssertJobDefinitionsEqual(streamingMapReduceJobDefinition, streamingMapReduceJobFromPowershell);
}
}
/// <inheritdoc />
public Task <JobCreationResults> CreateStreamingJobAsync(StreamingMapReduceJobCreateParameters streamingMapReduceJobCreateParameters)
{
var factory = ServiceLocator.Instance.Locate <IRemoteHadoopJobSubmissionPocoClientFactory>();
var pocoClient = factory.Create(this.credentials, this.Context, this.IgnoreSslErrors, this.GetCustomUserAgent());
return(pocoClient.SubmitStreamingJob(streamingMapReduceJobCreateParameters));
}
public Task<JobCreationResults> SubmitStreamingJob(StreamingMapReduceJobCreateParameters pigJobCreateParameters)
{
throw new System.NotImplementedException();
}
public void ICanCallThe_New_HDInsightStreamingMapReduceJobDefinitionCmdlet_WithArguments()
{
var streamingMapReduceJobDefinition = new StreamingMapReduceJobCreateParameters
{
JobName = "pi estimation jobDetails",
Input = TestConstants.WabsProtocolSchemeName + "input",
Output = TestConstants.WabsProtocolSchemeName + "input",
Mapper = TestConstants.WabsProtocolSchemeName + "combiner",
Reducer = TestConstants.WabsProtocolSchemeName + "combiner",
StatusFolder = TestConstants.WabsProtocolSchemeName + "someotherlocation"
};
streamingMapReduceJobDefinition.Arguments.Add("16");
streamingMapReduceJobDefinition.Arguments.Add("10000");
using (IRunspace runspace = this.GetPowerShellRunspace())
{
IPipelineResult results =
runspace.NewPipeline()
.AddCommand(CmdletConstants.NewAzureHDInsightStreamingMapReduceJobDefinition)
.WithParameter(CmdletConstants.JobName, streamingMapReduceJobDefinition.JobName)
.WithParameter(CmdletConstants.Input, streamingMapReduceJobDefinition.Input)
.WithParameter(CmdletConstants.Output, streamingMapReduceJobDefinition.Output)
.WithParameter(CmdletConstants.Mapper, streamingMapReduceJobDefinition.Mapper)
.WithParameter(CmdletConstants.Reducer, streamingMapReduceJobDefinition.Reducer)
.WithParameter(CmdletConstants.StatusFolder, streamingMapReduceJobDefinition.StatusFolder)
.WithParameter(CmdletConstants.Arguments, streamingMapReduceJobDefinition.Arguments)
.Invoke();
Assert.AreEqual(1, results.Results.Count);
AzureHDInsightStreamingMapReduceJobDefinition streamingMapReduceJobFromPowershell =
results.Results.ToEnumerable<AzureHDInsightStreamingMapReduceJobDefinition>().First();
AssertJobDefinitionsEqual(streamingMapReduceJobDefinition, streamingMapReduceJobFromPowershell);
foreach (string argument in streamingMapReduceJobDefinition.Arguments)
{
Assert.IsTrue(
streamingMapReduceJobFromPowershell.Arguments.Any(arg => string.Equals(argument, arg)),
"Unable to find argument '{0}' in value returned from powershell",
argument);
}
}
}
internal static void AssertJobDefinitionsEqual(
StreamingMapReduceJobCreateParameters streamingMapReduceJobDefinition,
AzureHDInsightStreamingMapReduceJobDefinition streamingMapReduceJobFromPowershell)
{
Assert.AreEqual(streamingMapReduceJobDefinition.JobName, streamingMapReduceJobFromPowershell.JobName);
Assert.AreEqual(streamingMapReduceJobDefinition.Input, streamingMapReduceJobFromPowershell.Input);
Assert.AreEqual(streamingMapReduceJobDefinition.Output, streamingMapReduceJobFromPowershell.Output);
Assert.AreEqual(streamingMapReduceJobDefinition.Mapper, streamingMapReduceJobFromPowershell.Mapper);
Assert.AreEqual(streamingMapReduceJobDefinition.Reducer, streamingMapReduceJobFromPowershell.Reducer);
Assert.AreEqual(streamingMapReduceJobDefinition.StatusFolder, streamingMapReduceJobFromPowershell.StatusFolder);
}
public Task<JobCreationResults> SubmitStreamingJob(StreamingMapReduceJobCreateParameters streamingMapReduceJobCreateParameters)
{
if (streamingMapReduceJobCreateParameters == null)
{
throw new ArgumentNullException("streamingMapReduceJobCreateParameters");
}
return Task.FromResult(this.CreateJobSuccessResult(streamingMapReduceJobCreateParameters, streamingMapReduceJobCreateParameters.JobName));
}
public void CanSerializeValidStreamingMapReduceJobRequest_WithNoReducer()
{
var streamingMapReduceJob = new StreamingMapReduceJobCreateParameters()
{
Input = Constants.WabsProtocolSchemeName + "input",
Output = Constants.WabsProtocolSchemeName + "output",
Mapper = Constants.WabsProtocolSchemeName + "mapper"
};
var payloadConverter = new PayloadConverterBase();
var payload = payloadConverter.SerializeStreamingMapReduceRequest("hadoop", streamingMapReduceJob);
Assert.IsTrue(payload.Contains(string.Format("{0}={1}", WebHCatConstants.Input, Uri.EscapeDataString(streamingMapReduceJob.Input))));
Assert.IsTrue(payload.Contains(string.Format("{0}={1}", WebHCatConstants.Output, Uri.EscapeDataString(streamingMapReduceJob.Output))));
Assert.IsTrue(payload.Contains(string.Format("{0}={1}", WebHCatConstants.Mapper, Uri.EscapeDataString(streamingMapReduceJob.Mapper))));
}
public void CanSerializeValidStreamingMapReduceJobRequest_WithFiles()
{
var streamingMapReduceJob = new StreamingMapReduceJobCreateParameters()
{
Input = Constants.WabsProtocolSchemeName + "input",
Output = Constants.WabsProtocolSchemeName + "output",
Mapper = Constants.WabsProtocolSchemeName + "mapper",
Reducer = Constants.WabsProtocolSchemeName + "reducer"
};
var resourceFile1 = "asv://container@hostname/myfile1";
var resourceFile2 = "asv://container@hostname/myfile2";
var payloadConverter = new PayloadConverterBase();
streamingMapReduceJob.Files.Add(resourceFile1);
streamingMapReduceJob.Files.Add(resourceFile2);
var payload = payloadConverter.SerializeStreamingMapReduceRequest("hadoop", streamingMapReduceJob);
Assert.IsTrue(payload.Contains(string.Format("{0}={1}", WebHCatConstants.Input, Uri.EscapeDataString(streamingMapReduceJob.Input))));
Assert.IsTrue(payload.Contains(string.Format("{0}={1}", WebHCatConstants.Output, Uri.EscapeDataString(streamingMapReduceJob.Output))));
Assert.IsTrue(payload.Contains(string.Format("{0}={1}", WebHCatConstants.Mapper, Uri.EscapeDataString(streamingMapReduceJob.Mapper))));
Assert.IsTrue(payload.Contains(string.Format("{0}={1}", WebHCatConstants.Reducer, Uri.EscapeDataString(streamingMapReduceJob.Reducer))));
Assert.IsTrue(payload.Contains(Uri.EscapeDataString(resourceFile1)));
Assert.IsTrue(payload.Contains(Uri.EscapeDataString(resourceFile2)));
}
public void CanSerializeValidStreamingMapReduceJobRequest_CmdEnv()
{
var streamingMapReduceJob = new StreamingMapReduceJobCreateParameters()
{
Input = "asv://input",
Output = "asv://output",
Mapper = "asv://mapper",
Reducer = "asv://reducer"
};
streamingMapReduceJob.CommandEnvironment.Add("Name1=Value1");
streamingMapReduceJob.CommandEnvironment.Add("Name2=Value2");
var payloadConverter = new PayloadConverterBase();
var payload = payloadConverter.SerializeStreamingMapReduceRequest("hadoop", streamingMapReduceJob);
Assert.IsTrue(payload.Contains(string.Format("{0}={1}", WebHCatConstants.Input, Uri.EscapeDataString(streamingMapReduceJob.Input))));
Assert.IsTrue(payload.Contains(string.Format("{0}={1}", WebHCatConstants.Output, Uri.EscapeDataString(streamingMapReduceJob.Output))));
Assert.IsTrue(payload.Contains(string.Format("{0}={1}", WebHCatConstants.Mapper, Uri.EscapeDataString(streamingMapReduceJob.Mapper))));
Assert.IsTrue(payload.Contains(string.Format("{0}={1}", WebHCatConstants.Reducer, Uri.EscapeDataString(streamingMapReduceJob.Reducer))));
foreach (var cmdEnvArgument in streamingMapReduceJob.CommandEnvironment)
{
Assert.IsTrue(payload.Contains(string.Format("{0}={1}", WebHCatConstants.CmdEnv, Uri.EscapeDataString(cmdEnvArgument))));
}
}
public void CanSerializeValidStreamingMapReduceJobRequest_Defines()
{
var streamingMapReduceJob = new StreamingMapReduceJobCreateParameters()
{
Input = "asv://input",
Output = "asv://output",
Mapper = "asv://mapper",
Reducer = "asv://reducer",
};
streamingMapReduceJob.Defines.Add(new KeyValuePair<string, string>("definekey1", "definevalue1"));
streamingMapReduceJob.Defines.Add(new KeyValuePair<string, string>("definekey2", "definevalue2"));
streamingMapReduceJob.Defines.Add(new KeyValuePair<string, string>("definekey3", "definevalue3"));
var payloadConverter = new PayloadConverterBase();
var payload = payloadConverter.SerializeStreamingMapReduceRequest("hadoop", streamingMapReduceJob);
Assert.IsTrue(payload.Contains(string.Format("{0}={1}", WebHCatConstants.Input, Uri.EscapeDataString(streamingMapReduceJob.Input))));
Assert.IsTrue(payload.Contains(string.Format("{0}={1}", WebHCatConstants.Output, Uri.EscapeDataString(streamingMapReduceJob.Output))));
Assert.IsTrue(payload.Contains(string.Format("{0}={1}", WebHCatConstants.Mapper, Uri.EscapeDataString(streamingMapReduceJob.Mapper))));
Assert.IsTrue(payload.Contains(string.Format("{0}={1}", WebHCatConstants.Reducer, Uri.EscapeDataString(streamingMapReduceJob.Reducer))));
foreach (var define in streamingMapReduceJob.Defines)
{
Assert.IsTrue(payload.Contains(Uri.EscapeDataString(string.Format("{0}={1}", define.Key, define.Value))));
}
}
public void CanSerializeValidStreamingMapReduceJobRequest()
{
var streamingMapReduceJob = new StreamingMapReduceJobCreateParameters()
{
Input = "asv://input",
Output = "asv://output",
Mapper = "asv://mapper",
Reducer = "asv://reducer",
Combiner = "asv://combiner"
};
var payloadConverter = new PayloadConverterBase();
var payload = payloadConverter.SerializeStreamingMapReduceRequest("hadoop", streamingMapReduceJob);
Assert.IsTrue(payload.Contains(string.Format("{0}={1}", WebHCatConstants.Input, Uri.EscapeDataString(streamingMapReduceJob.Input))));
Assert.IsTrue(payload.Contains(string.Format("{0}={1}", WebHCatConstants.Output, Uri.EscapeDataString(streamingMapReduceJob.Output))));
Assert.IsTrue(payload.Contains(string.Format("{0}={1}", WebHCatConstants.Mapper, Uri.EscapeDataString(streamingMapReduceJob.Mapper))));
Assert.IsTrue(payload.Contains(string.Format("{0}={1}", WebHCatConstants.Reducer, Uri.EscapeDataString(streamingMapReduceJob.Reducer))));
Assert.IsTrue(payload.Contains(string.Format("{0}={1}", WebHCatConstants.Combiner, Uri.EscapeDataString(streamingMapReduceJob.Combiner))));
}
/// <inheritdoc />
public JobCreationResults CreateStreamingJob(StreamingMapReduceJobCreateParameters streamingMapReduceJobCreateParameters)
{
return(this.CreateStreamingJobAsync(streamingMapReduceJobCreateParameters).WaitForResult());
}
public void CanCreateNewStreamingJob_StartJob()
{
var streamingMapReduceJobDefinition = new StreamingMapReduceJobCreateParameters
{
JobName = "pi estimation jobDetails",
Input = TestConstants.WabsProtocolSchemeName + "container@hostname/input",
Output = TestConstants.WabsProtocolSchemeName + "container@hostname/input",
Mapper = TestConstants.WabsProtocolSchemeName + "container@hostname/combiner",
Reducer = TestConstants.WabsProtocolSchemeName + "container@hostname/combiner",
StatusFolder = TestConstants.WabsProtocolSchemeName + "container@hostname/someotherlocation"
};
INewAzureHDInsightStreamingJobDefinitionCommand newStreamingMapReduceJobDefinitionCommand =
ServiceLocator.Instance.Locate<IAzureHDInsightCommandFactory>().CreateNewStreamingMapReduceDefinition();
newStreamingMapReduceJobDefinitionCommand.JobName = streamingMapReduceJobDefinition.JobName;
newStreamingMapReduceJobDefinitionCommand.JobName = streamingMapReduceJobDefinition.JobName;
newStreamingMapReduceJobDefinitionCommand.InputPath = streamingMapReduceJobDefinition.Input;
newStreamingMapReduceJobDefinitionCommand.OutputPath = streamingMapReduceJobDefinition.Output;
newStreamingMapReduceJobDefinitionCommand.Mapper = streamingMapReduceJobDefinition.Mapper;
newStreamingMapReduceJobDefinitionCommand.Reducer = streamingMapReduceJobDefinition.Reducer;
newStreamingMapReduceJobDefinitionCommand.StatusFolder = streamingMapReduceJobDefinition.StatusFolder;
newStreamingMapReduceJobDefinitionCommand.EndProcessing();
AzureHDInsightStreamingMapReduceJobDefinition streamingJobFromCommand = newStreamingMapReduceJobDefinitionCommand.Output.ElementAt(0);
TestJobStart(streamingJobFromCommand);
}
public JobCreationResults CreateStreamingJob(StreamingMapReduceJobCreateParameters streamingMapReduceJobCreateParameters)
{
return this.CreateStreamingJobAsync(streamingMapReduceJobCreateParameters).WaitForResult();
}
internal StreamingMapReduceJobCreateParameters ToStreamingMapReduceJobCreateParameters()
{
var streamingJobDefinition = new StreamingMapReduceJobCreateParameters
{
JobName = this.JobName,
Input = this.Input,
Output = this.Output,
Reducer = this.Reducer,
Combiner = this.Combiner,
Mapper = this.Mapper,
StatusFolder = this.StatusFolder
};
streamingJobDefinition.CommandEnvironment.AddRange(this.CommandEnvironment);
streamingJobDefinition.Arguments.AddRange(this.Arguments);
streamingJobDefinition.Defines.AddRange(this.Defines);
streamingJobDefinition.Files.AddRange(this.Files);
return streamingJobDefinition;
}
/// <inheritdoc />
public async Task<JobCreationResults> SubmitStreamingJob(StreamingMapReduceJobCreateParameters pigJobCreateParameters)
{
var remoteClient = ServiceLocator.Instance.Locate<IRemoteHadoopJobSubmissionPocoClientFactory>().Create(this.remoteCreds, this.context, this.ignoreSslErrors, this.GetUserAgentString());
return await remoteClient.SubmitStreamingJob(pigJobCreateParameters);
}