public NewAzureHDInsightStreamingMapReduceJobDefinitionCommand()
{
Arguments = new string[] {};
CommandEnvironment = new string[] {};
Defines = new Hashtable();
job = new AzureHDInsightStreamingMapReduceJobDefinition();
}
public virtual void ICanCallThe_NewStreamingJob_Then_Start_HDInsightJobsCmdlet()
{
var streamingMapReduceJobDefinition = new StreamingMapReduceJobCreateParameters
{
JobName = "pi estimation jobDetails",
Input = TestConstants.WabsProtocolSchemeName + "input",
Output = TestConstants.WabsProtocolSchemeName + "input",
Mapper = TestConstants.WabsProtocolSchemeName + "combiner",
Reducer = TestConstants.WabsProtocolSchemeName + "combiner",
StatusFolder = TestConstants.WabsProtocolSchemeName + "someotherlocation"
};
using (IRunspace runspace = this.GetPowerShellRunspace())
{
IPipelineResult results =
runspace.NewPipeline()
.AddCommand(CmdletConstants.NewAzureHDInsightStreamingMapReduceJobDefinition)
.WithParameter(CmdletConstants.JobName, streamingMapReduceJobDefinition.JobName)
.WithParameter(CmdletConstants.Input, streamingMapReduceJobDefinition.Input)
.WithParameter(CmdletConstants.Output, streamingMapReduceJobDefinition.Output)
.WithParameter(CmdletConstants.Mapper, streamingMapReduceJobDefinition.Mapper)
.WithParameter(CmdletConstants.Reducer, streamingMapReduceJobDefinition.Reducer)
.WithParameter(CmdletConstants.StatusFolder, streamingMapReduceJobDefinition.StatusFolder)
.Invoke();
Assert.AreEqual(1, results.Results.Count);
AzureHDInsightStreamingMapReduceJobDefinition streamingJobFromPowershell =
results.Results.ToEnumerable <AzureHDInsightStreamingMapReduceJobDefinition>().First();
RunJobInPowershell(runspace, streamingJobFromPowershell);
}
}
public virtual void NewStreamingMapReduceJob_StartJob_GetJob()
{
var streamingMapReduceJobDefinition = new StreamingMapReduceJobCreateParameters
{
JobName = "environment variable jobDetails",
Mapper = "environmentvariables.exe",
Input = "/example/apps/environmentvariables.exe",
Output = Guid.NewGuid().ToString()
};
var files = new List <string> {
streamingMapReduceJobDefinition.Mapper
};
using (IRunspace runspace = this.GetPowerShellRunspace())
{
IPipelineResult results =
runspace.NewPipeline()
.AddCommand(CmdletConstants.NewAzureHDInsightStreamingMapReduceJobDefinition)
.WithParameter(CmdletConstants.JobName, streamingMapReduceJobDefinition.JobName)
.WithParameter(CmdletConstants.Mapper, streamingMapReduceJobDefinition.Mapper)
.WithParameter(CmdletConstants.Input, streamingMapReduceJobDefinition.Input)
.WithParameter(CmdletConstants.Output, streamingMapReduceJobDefinition.Output)
.WithParameter(CmdletConstants.Files, files)
.Invoke();
Assert.AreEqual(1, results.Results.Count);
AzureHDInsightStreamingMapReduceJobDefinition streamingMapReduceJobFromPowershell =
results.Results.ToEnumerable <AzureHDInsightStreamingMapReduceJobDefinition>().First();
RunJobAndGetWithId(runspace, streamingMapReduceJobFromPowershell);
}
}
public void CanCreateNewStreamingMapReduceDefinition_WithArguments()
{
StreamingMapReduceJobCreateParameters streamingMapReduceJobDefinition = this.GetStreamingMapReduceJob();
streamingMapReduceJobDefinition.Arguments.Add("16");
streamingMapReduceJobDefinition.Arguments.Add("10000");
INewAzureHDInsightStreamingJobDefinitionCommand newStreamingMapReduceJobDefinitionCommand =
ServiceLocator.Instance.Locate <IAzureHDInsightCommandFactory>().CreateNewStreamingMapReduceDefinition();
newStreamingMapReduceJobDefinitionCommand.JobName = streamingMapReduceJobDefinition.JobName;
newStreamingMapReduceJobDefinitionCommand.JobName = streamingMapReduceJobDefinition.JobName;
newStreamingMapReduceJobDefinitionCommand.InputPath = streamingMapReduceJobDefinition.Input;
newStreamingMapReduceJobDefinitionCommand.OutputPath = streamingMapReduceJobDefinition.Output;
newStreamingMapReduceJobDefinitionCommand.Mapper = streamingMapReduceJobDefinition.Mapper;
newStreamingMapReduceJobDefinitionCommand.Reducer = streamingMapReduceJobDefinition.Reducer;
newStreamingMapReduceJobDefinitionCommand.StatusFolder = streamingMapReduceJobDefinition.StatusFolder;
newStreamingMapReduceJobDefinitionCommand.Arguments = streamingMapReduceJobDefinition.Arguments.ToArray();
newStreamingMapReduceJobDefinitionCommand.EndProcessing();
AzureHDInsightStreamingMapReduceJobDefinition streamingMapReduceJobFromCommand =
newStreamingMapReduceJobDefinitionCommand.Output.ElementAt(0);
NewStreamingMapReduceJobCmdLetTests.AssertJobDefinitionsEqual(streamingMapReduceJobDefinition, streamingMapReduceJobFromCommand);
foreach (string argument in streamingMapReduceJobDefinition.Arguments)
{
Assert.IsTrue(
streamingMapReduceJobFromCommand.Arguments.Any(arg => string.Equals(argument, arg)),
"Unable to find argument '{0}' in value returned from command",
argument);
}
}
public void CanCreateNewStreamingJob_StartJob()
{
var streamingMapReduceJobDefinition = new StreamingMapReduceJobCreateParameters
{
JobName = "pi estimation jobDetails",
Input = TestConstants.WabsProtocolSchemeName + "container@hostname/input",
Output = TestConstants.WabsProtocolSchemeName + "container@hostname/input",
Mapper = TestConstants.WabsProtocolSchemeName + "container@hostname/combiner",
Reducer = TestConstants.WabsProtocolSchemeName + "container@hostname/combiner",
StatusFolder = TestConstants.WabsProtocolSchemeName + "container@hostname/someotherlocation"
};
INewAzureHDInsightStreamingJobDefinitionCommand newStreamingMapReduceJobDefinitionCommand =
ServiceLocator.Instance.Locate <IAzureHDInsightCommandFactory>().CreateNewStreamingMapReduceDefinition();
newStreamingMapReduceJobDefinitionCommand.JobName = streamingMapReduceJobDefinition.JobName;
newStreamingMapReduceJobDefinitionCommand.JobName = streamingMapReduceJobDefinition.JobName;
newStreamingMapReduceJobDefinitionCommand.InputPath = streamingMapReduceJobDefinition.Input;
newStreamingMapReduceJobDefinitionCommand.OutputPath = streamingMapReduceJobDefinition.Output;
newStreamingMapReduceJobDefinitionCommand.Mapper = streamingMapReduceJobDefinition.Mapper;
newStreamingMapReduceJobDefinitionCommand.Reducer = streamingMapReduceJobDefinition.Reducer;
newStreamingMapReduceJobDefinitionCommand.StatusFolder = streamingMapReduceJobDefinition.StatusFolder;
newStreamingMapReduceJobDefinitionCommand.EndProcessing();
AzureHDInsightStreamingMapReduceJobDefinition streamingJobFromCommand = newStreamingMapReduceJobDefinitionCommand.Output.ElementAt(0);
TestJobStart(streamingJobFromCommand);
}
internal static void AssertJobDefinitionsEqual(
StreamingMapReduceJobCreateParameters streamingMapReduceJobDefinition,
AzureHDInsightStreamingMapReduceJobDefinition streamingMapReduceJobFromPowershell)
{
Assert.AreEqual(streamingMapReduceJobDefinition.JobName, streamingMapReduceJobFromPowershell.JobName);
Assert.AreEqual(streamingMapReduceJobDefinition.Input, streamingMapReduceJobFromPowershell.Input);
Assert.AreEqual(streamingMapReduceJobDefinition.Output, streamingMapReduceJobFromPowershell.Output);
Assert.AreEqual(streamingMapReduceJobDefinition.Mapper, streamingMapReduceJobFromPowershell.Mapper);
Assert.AreEqual(streamingMapReduceJobDefinition.Reducer, streamingMapReduceJobFromPowershell.Reducer);
Assert.AreEqual(streamingMapReduceJobDefinition.StatusFolder, streamingMapReduceJobFromPowershell.StatusFolder);
}
public void CanCreateNewStreamingMapReduceJob_WithoutJobName_FilesRelative()
{
var streamingMapReduceJobDefinition = new AzureHDInsightStreamingMapReduceJobDefinition
{
Input = "input",
Output = "output",
Mapper = "/examples/mapper.exe",
Reducer = "/examples/combiner.exe",
StatusFolder = "/someotherlocation"
};
AzureHDInsightJob startedJob = TestJobStart(streamingMapReduceJobDefinition);
Assert.AreEqual("mapper.exe", startedJob.Name);
}
public void CanCreateNewStreamingMapReduceJob_WithoutJobName()
{
var streamingMapReduceJobDefinition = new AzureHDInsightStreamingMapReduceJobDefinition
{
Input = TestConstants.WabsProtocolSchemeName + "container@hostname/input",
Output = TestConstants.WabsProtocolSchemeName + "container@hostname/input",
Mapper = TestConstants.WabsProtocolSchemeName + "container@hostname/mapper",
Reducer = TestConstants.WabsProtocolSchemeName + "container@hostname/combiner",
StatusFolder = TestConstants.WabsProtocolSchemeName + "container@hostname/someotherlocation"
};
AzureHDInsightJob startedJob = TestJobStart(streamingMapReduceJobDefinition);
Assert.AreEqual(startedJob.Name, "mapper");
}
public void ICanCallThe_New_HDInsightStreamingMapReduceJobDefinitionCmdlet_WithParameters()
{
var streamingMapReduceJobDefinition = new StreamingMapReduceJobCreateParameters
{
JobName = "pi estimation jobDetails",
Input = TestConstants.WabsProtocolSchemeName + "input",
Output = TestConstants.WabsProtocolSchemeName + "input",
Mapper = TestConstants.WabsProtocolSchemeName + "combiner",
Reducer = TestConstants.WabsProtocolSchemeName + "combiner",
StatusFolder = TestConstants.WabsProtocolSchemeName + "someotherlocation"
};
streamingMapReduceJobDefinition.Defines.Add("map.input.tasks", "1000");
streamingMapReduceJobDefinition.Defines.Add("map.input.reducers", "1000");
using (IRunspace runspace = this.GetPowerShellRunspace())
{
IPipelineResult results =
runspace.NewPipeline()
.AddCommand(CmdletConstants.NewAzureHDInsightStreamingMapReduceJobDefinition)
.WithParameter(CmdletConstants.JobName, streamingMapReduceJobDefinition.JobName)
.WithParameter(CmdletConstants.Input, streamingMapReduceJobDefinition.Input)
.WithParameter(CmdletConstants.Output, streamingMapReduceJobDefinition.Output)
.WithParameter(CmdletConstants.Mapper, streamingMapReduceJobDefinition.Mapper)
.WithParameter(CmdletConstants.Reducer, streamingMapReduceJobDefinition.Reducer)
.WithParameter(CmdletConstants.StatusFolder, streamingMapReduceJobDefinition.StatusFolder)
.WithParameter(CmdletConstants.Parameters, streamingMapReduceJobDefinition.Defines)
.Invoke();
Assert.AreEqual(1, results.Results.Count);
AzureHDInsightStreamingMapReduceJobDefinition streamingMapReduceJobFromPowershell =
results.Results.ToEnumerable <AzureHDInsightStreamingMapReduceJobDefinition>().First();
AssertJobDefinitionsEqual(streamingMapReduceJobDefinition, streamingMapReduceJobFromPowershell);
foreach (var parameter in streamingMapReduceJobDefinition.Defines)
{
Assert.IsTrue(
streamingMapReduceJobFromPowershell.Defines.Any(
arg => string.Equals(parameter.Key, arg.Key) && string.Equals(parameter.Value, arg.Value)),
"Unable to find parameter '{0}' in value returned from powershell",
parameter.Key);
}
}
}
public void CanCreateNewStreamingMapReduceDefinition()
{
StreamingMapReduceJobCreateParameters streamingMapReduceJobDefinition = this.GetStreamingMapReduceJob();
INewAzureHDInsightStreamingJobDefinitionCommand newStreamingMapReduceJobDefinitionCommand =
ServiceLocator.Instance.Locate <IAzureHDInsightCommandFactory>().CreateNewStreamingMapReduceDefinition();
newStreamingMapReduceJobDefinitionCommand.JobName = streamingMapReduceJobDefinition.JobName;
newStreamingMapReduceJobDefinitionCommand.JobName = streamingMapReduceJobDefinition.JobName;
newStreamingMapReduceJobDefinitionCommand.InputPath = streamingMapReduceJobDefinition.Input;
newStreamingMapReduceJobDefinitionCommand.OutputPath = streamingMapReduceJobDefinition.Output;
newStreamingMapReduceJobDefinitionCommand.Mapper = streamingMapReduceJobDefinition.Mapper;
newStreamingMapReduceJobDefinitionCommand.Reducer = streamingMapReduceJobDefinition.Reducer;
newStreamingMapReduceJobDefinitionCommand.StatusFolder = streamingMapReduceJobDefinition.StatusFolder;
newStreamingMapReduceJobDefinitionCommand.EndProcessing();
AzureHDInsightStreamingMapReduceJobDefinition streamingMapReduceJobFromCommand =
newStreamingMapReduceJobDefinitionCommand.Output.ElementAt(0);
NewStreamingMapReduceJobCmdLetTests.AssertJobDefinitionsEqual(streamingMapReduceJobDefinition, streamingMapReduceJobFromCommand);
}
public void CanCreateNewStreamingMapReduceDefinition_WithParameters()
{
StreamingMapReduceJobCreateParameters streamingMapReduceJobDefinition = this.GetStreamingMapReduceJob();
streamingMapReduceJobDefinition.Defines.Add("map.input.tasks", "1000");
streamingMapReduceJobDefinition.Defines.Add("map.input.reducers", "1000");
INewAzureHDInsightStreamingJobDefinitionCommand newStreamingMapReduceJobDefinitionCommand =
ServiceLocator.Instance.Locate <IAzureHDInsightCommandFactory>().CreateNewStreamingMapReduceDefinition();
newStreamingMapReduceJobDefinitionCommand.JobName = streamingMapReduceJobDefinition.JobName;
newStreamingMapReduceJobDefinitionCommand.JobName = streamingMapReduceJobDefinition.JobName;
newStreamingMapReduceJobDefinitionCommand.InputPath = streamingMapReduceJobDefinition.Input;
newStreamingMapReduceJobDefinitionCommand.OutputPath = streamingMapReduceJobDefinition.Output;
newStreamingMapReduceJobDefinitionCommand.Mapper = streamingMapReduceJobDefinition.Mapper;
newStreamingMapReduceJobDefinitionCommand.Reducer = streamingMapReduceJobDefinition.Reducer;
newStreamingMapReduceJobDefinitionCommand.StatusFolder = streamingMapReduceJobDefinition.StatusFolder;
newStreamingMapReduceJobDefinitionCommand.Defines = new Hashtable();
foreach (var define in streamingMapReduceJobDefinition.Defines)
{
newStreamingMapReduceJobDefinitionCommand.Defines.Add(define.Key, define.Value);
}
newStreamingMapReduceJobDefinitionCommand.EndProcessing();
AzureHDInsightStreamingMapReduceJobDefinition streamingMapReduceJobFromCommand =
newStreamingMapReduceJobDefinitionCommand.Output.ElementAt(0);
NewStreamingMapReduceJobCmdLetTests.AssertJobDefinitionsEqual(streamingMapReduceJobDefinition, streamingMapReduceJobFromCommand);
foreach (var parameter in streamingMapReduceJobDefinition.Defines)
{
Assert.IsTrue(
streamingMapReduceJobFromCommand.Defines.Any(
arg => string.Equals(parameter.Key, arg.Key) && string.Equals(parameter.Value, arg.Value)),
"Unable to find parameter '{0}' in value returned from command",
parameter.Key);
}
}
private static async Task <JobCreationResults> CreateStreamingJob(
AzureHDInsightStreamingMapReduceJobDefinition azureStreamingJobDefinition, IJobSubmissionClient client)
{
var streamingJobDefinition = new StreamingMapReduceJobCreateParameters
{
JobName = azureStreamingJobDefinition.JobName,
Input = azureStreamingJobDefinition.Input,
Output = azureStreamingJobDefinition.Output,
Reducer = azureStreamingJobDefinition.Reducer,
Combiner = azureStreamingJobDefinition.Combiner,
Mapper = azureStreamingJobDefinition.Mapper
};
streamingJobDefinition.StatusFolder = azureStreamingJobDefinition.StatusFolder;
streamingJobDefinition.CommandEnvironment.AddRange(azureStreamingJobDefinition.CommandEnvironment);
streamingJobDefinition.Arguments.AddRange(azureStreamingJobDefinition.Arguments);
streamingJobDefinition.Defines.AddRange(azureStreamingJobDefinition.Defines);
streamingJobDefinition.Files.AddRange(azureStreamingJobDefinition.Files);
var jobCreationResults = await client.CreateStreamingJobAsync(streamingJobDefinition);
return(jobCreationResults);
}
public void CanCreateNewStreamingMapReduceJob_WithoutJobName_FilesRelative()
{
var streamingMapReduceJobDefinition = new AzureHDInsightStreamingMapReduceJobDefinition
{
Input = "input",
Output = "output",
Mapper = "/examples/mapper.exe",
Reducer = "/examples/combiner.exe",
StatusFolder = "/someotherlocation"
};
AzureHDInsightJob startedJob = TestJobStart(streamingMapReduceJobDefinition);
Assert.AreEqual("mapper.exe", startedJob.Name);
}
public void CanCreateNewStreamingMapReduceJob_WithoutJobName()
{
var streamingMapReduceJobDefinition = new AzureHDInsightStreamingMapReduceJobDefinition
{
Input = TestConstants.WabsProtocolSchemeName + "container@hostname/input",
Output = TestConstants.WabsProtocolSchemeName + "container@hostname/input",
Mapper = TestConstants.WabsProtocolSchemeName + "container@hostname/mapper",
Reducer = TestConstants.WabsProtocolSchemeName + "container@hostname/combiner",
StatusFolder = TestConstants.WabsProtocolSchemeName + "container@hostname/someotherlocation"
};
AzureHDInsightJob startedJob = TestJobStart(streamingMapReduceJobDefinition);
Assert.AreEqual(startedJob.Name, "mapper");
}
internal static void AssertJobDefinitionsEqual(
StreamingMapReduceJobCreateParameters streamingMapReduceJobDefinition,
AzureHDInsightStreamingMapReduceJobDefinition streamingMapReduceJobFromPowershell)
{
Assert.AreEqual(streamingMapReduceJobDefinition.JobName, streamingMapReduceJobFromPowershell.JobName);
Assert.AreEqual(streamingMapReduceJobDefinition.Input, streamingMapReduceJobFromPowershell.Input);
Assert.AreEqual(streamingMapReduceJobDefinition.Output, streamingMapReduceJobFromPowershell.Output);
Assert.AreEqual(streamingMapReduceJobDefinition.Mapper, streamingMapReduceJobFromPowershell.Mapper);
Assert.AreEqual(streamingMapReduceJobDefinition.Reducer, streamingMapReduceJobFromPowershell.Reducer);
Assert.AreEqual(streamingMapReduceJobDefinition.StatusFolder, streamingMapReduceJobFromPowershell.StatusFolder);
}
