public void CanCreateNewMapReduceDefinition_WithArguments()
{
var mapReduceJobDefinition = new MapReduceJobCreateParameters
{
JobName = "pi estimation jobDetails",
ClassName = "pi",
JarFile = TestConstants.WabsProtocolSchemeName + "container@hostname/examples.jar"
};
mapReduceJobDefinition.Arguments.Add("16");
mapReduceJobDefinition.Arguments.Add("10000");
INewAzureHDInsightMapReduceJobDefinitionCommand newMapReduceJobDefinitionCommand =
ServiceLocator.Instance.Locate <IAzureHDInsightCommandFactory>().CreateNewMapReduceDefinition();
newMapReduceJobDefinitionCommand.JobName = mapReduceJobDefinition.JobName;
newMapReduceJobDefinitionCommand.JarFile = mapReduceJobDefinition.JarFile;
newMapReduceJobDefinitionCommand.ClassName = mapReduceJobDefinition.ClassName;
newMapReduceJobDefinitionCommand.Arguments = mapReduceJobDefinition.Arguments.ToArray();
newMapReduceJobDefinitionCommand.EndProcessing();
AzureHDInsightMapReduceJobDefinition mapReduceJobFromCommand = newMapReduceJobDefinitionCommand.Output.ElementAt(0);
Assert.AreEqual(mapReduceJobDefinition.JobName, mapReduceJobFromCommand.JobName);
Assert.AreEqual(mapReduceJobDefinition.ClassName, mapReduceJobFromCommand.ClassName);
Assert.AreEqual(mapReduceJobDefinition.JarFile, mapReduceJobFromCommand.JarFile);
foreach (string argument in mapReduceJobDefinition.Arguments)
{
Assert.IsTrue(
mapReduceJobFromCommand.Arguments.Any(arg => string.Equals(argument, arg)),
"Unable to find argument '{0}' in value returned from command",
argument);
}
}
public virtual void ICanCallThe_NewMapReduceJob_Then_Start_HDInsightJobsCmdlet()
{
var mapReduceJobDefinition = new MapReduceJobCreateParameters
{
JobName = "pi estimation jobDetails",
ClassName = "pi",
JarFile = "/example/hadoop-examples.jar"
};
mapReduceJobDefinition.Arguments.Add("16");
mapReduceJobDefinition.Arguments.Add("10000");
using (IRunspace runspace = this.GetPowerShellRunspace())
{
IPipelineResult results =
runspace.NewPipeline()
.AddCommand(CmdletConstants.NewAzureHDInsightMapReduceJobDefinition)
.WithParameter(CmdletConstants.JobName, mapReduceJobDefinition.JobName)
.WithParameter(CmdletConstants.JarFile, mapReduceJobDefinition.JarFile)
.WithParameter(CmdletConstants.ClassName, mapReduceJobDefinition.ClassName)
.WithParameter(CmdletConstants.Arguments, mapReduceJobDefinition.Arguments)
.Invoke();
Assert.AreEqual(1, results.Results.Count);
AzureHDInsightMapReduceJobDefinition mapReduceJobFromPowershell =
results.Results.ToEnumerable <AzureHDInsightMapReduceJobDefinition>().First();
RunJobInPowershell(runspace, mapReduceJobFromPowershell);
}
}
public void ICanCallThe_New_HDInsightMapReduceJobDefinitionCmdlet()
{
var mapReduceJobDefinition = new MapReduceJobCreateParameters
{
JobName = "pi estimation jobDetails",
ClassName = "pi",
JarFile = TestConstants.WabsProtocolSchemeName + "container@hostname/examples.jar",
};
mapReduceJobDefinition.LibJars.Add("some.jarfile.jar");
using (IRunspace runspace = this.GetPowerShellRunspace())
{
IPipelineResult results =
runspace.NewPipeline()
.AddCommand(CmdletConstants.NewAzureHDInsightMapReduceJobDefinition)
.WithParameter(CmdletConstants.JobName, mapReduceJobDefinition.JobName)
.WithParameter(CmdletConstants.JarFile, mapReduceJobDefinition.JarFile)
.WithParameter(CmdletConstants.ClassName, mapReduceJobDefinition.ClassName)
.WithParameter(CmdletConstants.LibJars, mapReduceJobDefinition.LibJars)
.Invoke();
Assert.AreEqual(1, results.Results.Count);
AzureHDInsightMapReduceJobDefinition mapReduceJobFromPowershell =
results.Results.ToEnumerable <AzureHDInsightMapReduceJobDefinition>().First();
Assert.AreEqual(mapReduceJobDefinition.JobName, mapReduceJobFromPowershell.JobName);
Assert.AreEqual(mapReduceJobDefinition.ClassName, mapReduceJobFromPowershell.ClassName);
Assert.AreEqual(mapReduceJobDefinition.JarFile, mapReduceJobFromPowershell.JarFile);
Assert.AreEqual(mapReduceJobDefinition.LibJars.Count, mapReduceJobFromPowershell.LibJars.Count);
Assert.AreEqual(mapReduceJobDefinition.LibJars.First(), mapReduceJobFromPowershell.LibJars.First());
}
}
public NewAzureHDInsightMapReduceJobDefinitionCommand()
{
Arguments = new string[] { };
Files = new string[] { };
Defines = new Hashtable();
LibJars = new string[] { };
job = new AzureHDInsightMapReduceJobDefinition();
}
private static async Task <JobCreationResults> SubmitMapReduceJob(
AzureHDInsightMapReduceJobDefinition azureMapReduceJobDefinition, IJobSubmissionClient client)
{
MapReduceJobCreateParameters mapReduceJobDefinition = azureMapReduceJobDefinition.ToMapReduceJobCreateParameters();
var jobCreationResults = await client.CreateMapReduceJobAsync(mapReduceJobDefinition);
return(jobCreationResults);
}
public void CanStartMapReduceJob()
{
var mapReduceJobDefinition = new AzureHDInsightMapReduceJobDefinition
{
JobName = "pi estimation jobDetails",
ClassName = "pi",
JarFile = TestConstants.WabsProtocolSchemeName + "container@hostname/examples.jar"
};
TestJobStart(mapReduceJobDefinition);
}
public void CanAutoGenerateStatusDirectoryForMapReduceJob()
{
var mapReduceJobDefinition = new AzureHDInsightMapReduceJobDefinition
{
JobName = "pi estimation jobDetails",
ClassName = "pi",
JarFile = TestConstants.WabsProtocolSchemeName + "container@hostname/examples.jar"
};
AzureHDInsightJob startedJob = TestJobStart(mapReduceJobDefinition);
Assert.IsFalse(string.IsNullOrEmpty(startedJob.StatusDirectory));
}
public virtual void ICanCallThe_Start_HDInsightJobsCmdlet_WithoutName()
{
var mapReduceJobDefinition = new AzureHDInsightMapReduceJobDefinition
{
ClassName = "pi",
JarFile = TestConstants.WabsProtocolSchemeName + "container@hostname/examples.jar"
};
using (IRunspace runspace = this.GetPowerShellRunspace())
{
RunJobInPowershell(runspace, mapReduceJobDefinition);
}
}
public void CanAutoGenerateStatusDirectoryForMapReduceJob()
{
var mapReduceJobDefinition = new AzureHDInsightMapReduceJobDefinition
{
JobName = "pi estimation jobDetails",
ClassName = "pi",
JarFile = TestConstants.WabsProtocolSchemeName + "container@hostname/examples.jar"
};
AzureHDInsightJob startedJob = TestJobStart(mapReduceJobDefinition);
Assert.IsFalse(string.IsNullOrEmpty(startedJob.StatusDirectory));
}
public void CanCreateNewMapReduceJob_WithoutJobName()
{
var mapReduceJobDefinition = new AzureHDInsightMapReduceJobDefinition
{
ClassName = "pi",
JarFile = TestConstants.WabsProtocolSchemeName + "container@hostname/examples.jar",
StatusFolder = "/myoutputfolder"
};
AzureHDInsightJob startedJob = TestJobStart(mapReduceJobDefinition);
Assert.AreEqual("pi", startedJob.Name);
}
public void UserCanSupplyStatusDirectoryForMapReduceJob()
{
var mapReduceJobDefinition = new AzureHDInsightMapReduceJobDefinition
{
JobName = "pi estimation jobDetails",
ClassName = "pi",
JarFile = TestConstants.WabsProtocolSchemeName + "container@hostname/examples.jar",
StatusFolder = "/myoutputfolder"
};
AzureHDInsightJob startedJob = TestJobStart(mapReduceJobDefinition);
Assert.AreEqual(startedJob.StatusDirectory, mapReduceJobDefinition.StatusFolder);
}
public void ICanCallThe_Start_HDInsightJobsCmdlet_WithDebug()
{
var mapReduceJobDefinition = new AzureHDInsightMapReduceJobDefinition
{
JobName = "pi estimation jobDetails",
ClassName = "pi",
JarFile = TestConstants.WabsProtocolSchemeName + "container@hostname/examples.jar"
};
using (IRunspace runspace = this.GetPowerShellRunspace())
{
string expectedLogMessage = string.Format(CultureInfo.InvariantCulture, "Starting jobDetails '{0}'.", mapReduceJobDefinition.JobName);
RunJobInPowershell(
runspace, mapReduceJobDefinition, CmdletScenariosTestCaseBase.GetHttpAccessEnabledCluster(), true, expectedLogMessage);
}
}
public void CanCreateNewMapReduceJob_StartJob()
{
var mapReduceJobDefinition = new MapReduceJobCreateParameters
{
JobName = "pi estimation jobDetails",
ClassName = "pi",
JarFile = TestConstants.WabsProtocolSchemeName + "container@hostname/examples.jar"
};
INewAzureHDInsightMapReduceJobDefinitionCommand newMapReduceJobDefinitionCommand =
ServiceLocator.Instance.Locate <IAzureHDInsightCommandFactory>().CreateNewMapReduceDefinition();
newMapReduceJobDefinitionCommand.JobName = mapReduceJobDefinition.JobName;
newMapReduceJobDefinitionCommand.JarFile = mapReduceJobDefinition.JarFile;
newMapReduceJobDefinitionCommand.ClassName = mapReduceJobDefinition.ClassName;
newMapReduceJobDefinitionCommand.EndProcessing();
AzureHDInsightMapReduceJobDefinition mapReduceJobFromCommand = newMapReduceJobDefinitionCommand.Output.ElementAt(0);
TestJobStart(mapReduceJobFromCommand);
}
public void ICanCallThe_New_HDInsightMapReduceJobDefinitionCmdlet_WithParameters()
{
var mapReduceJobDefinition = new MapReduceJobCreateParameters
{
JobName = "pi estimation jobDetails",
ClassName = "pi",
JarFile = TestConstants.WabsProtocolSchemeName + "container@hostname/examples.jar"
};
mapReduceJobDefinition.Defines.Add("map.input.tasks", "1000");
mapReduceJobDefinition.Defines.Add("map.input.reducers", "1000");
using (IRunspace runspace = this.GetPowerShellRunspace())
{
IPipelineResult results =
runspace.NewPipeline()
.AddCommand(CmdletConstants.NewAzureHDInsightMapReduceJobDefinition)
.WithParameter(CmdletConstants.JobName, mapReduceJobDefinition.JobName)
.WithParameter(CmdletConstants.JarFile, mapReduceJobDefinition.JarFile)
.WithParameter(CmdletConstants.ClassName, mapReduceJobDefinition.ClassName)
.WithParameter(CmdletConstants.Parameters, mapReduceJobDefinition.Defines)
.Invoke();
Assert.AreEqual(1, results.Results.Count);
AzureHDInsightMapReduceJobDefinition mapReduceJobFromPowershell =
results.Results.ToEnumerable <AzureHDInsightMapReduceJobDefinition>().First();
Assert.AreEqual(mapReduceJobDefinition.JobName, mapReduceJobFromPowershell.JobName);
Assert.AreEqual(mapReduceJobDefinition.ClassName, mapReduceJobFromPowershell.ClassName);
Assert.AreEqual(mapReduceJobDefinition.JarFile, mapReduceJobFromPowershell.JarFile);
foreach (var parameter in mapReduceJobDefinition.Defines)
{
Assert.IsTrue(
mapReduceJobFromPowershell.Defines.Any(
arg => string.Equals(parameter.Key, arg.Key) && string.Equals(parameter.Value, arg.Value)),
"Unable to find parameter '{0}' in value returned from powershell",
parameter.Key);
}
}
}
public void CanCreateNewMapReduceDefinition_WithParameters()
{
var mapReduceJobDefinition = new MapReduceJobCreateParameters
{
JobName = "pi estimation jobDetails",
ClassName = "pi",
JarFile = TestConstants.WabsProtocolSchemeName + "container@hostname/examples.jar"
};
mapReduceJobDefinition.Defines.Add("map.input.tasks", "1000");
mapReduceJobDefinition.Defines.Add("map.input.reducers", "1000");
INewAzureHDInsightMapReduceJobDefinitionCommand newMapReduceJobDefinitionCommand =
ServiceLocator.Instance.Locate <IAzureHDInsightCommandFactory>().CreateNewMapReduceDefinition();
newMapReduceJobDefinitionCommand.JobName = mapReduceJobDefinition.JobName;
newMapReduceJobDefinitionCommand.JarFile = mapReduceJobDefinition.JarFile;
newMapReduceJobDefinitionCommand.ClassName = mapReduceJobDefinition.ClassName;
foreach (var define in mapReduceJobDefinition.Defines)
{
newMapReduceJobDefinitionCommand.Defines.Add(define.Key, define.Value);
}
newMapReduceJobDefinitionCommand.EndProcessing();
AzureHDInsightMapReduceJobDefinition mapReduceJobFromCommand = newMapReduceJobDefinitionCommand.Output.ElementAt(0);
Assert.AreEqual(mapReduceJobDefinition.JobName, mapReduceJobFromCommand.JobName);
Assert.AreEqual(mapReduceJobDefinition.ClassName, mapReduceJobFromCommand.ClassName);
Assert.AreEqual(mapReduceJobDefinition.JarFile, mapReduceJobFromCommand.JarFile);
foreach (var parameter in mapReduceJobDefinition.Defines)
{
Assert.IsTrue(
mapReduceJobFromCommand.Defines.Any(arg => string.Equals(parameter.Key, arg.Key) && string.Equals(parameter.Value, arg.Value)),
"Unable to find parameter '{0}' in value returned from command",
parameter.Key);
}
}
public void ICanCallThe_Start_HDInsightJobsCmdlet_WithDebug()
{
var mapReduceJobDefinition = new AzureHDInsightMapReduceJobDefinition
{
JobName = "pi estimation jobDetails",
ClassName = "pi",
JarFile = TestConstants.WabsProtocolSchemeName + "container@hostname/examples.jar"
};
using (IRunspace runspace = this.GetPowerShellRunspace())
{
string expectedLogMessage = string.Format(CultureInfo.InvariantCulture, "Starting jobDetails '{0}'.", mapReduceJobDefinition.JobName);
RunJobInPowershell(
runspace, mapReduceJobDefinition, CmdletScenariosTestCaseBase.GetHttpAccessEnabledCluster(), true, expectedLogMessage);
}
}
public virtual void ICanCallThe_Start_HDInsightJobsCmdlet_WithoutName()
{
var mapReduceJobDefinition = new AzureHDInsightMapReduceJobDefinition
{
ClassName = "pi",
JarFile = TestConstants.WabsProtocolSchemeName + "container@hostname/examples.jar"
};
using (IRunspace runspace = this.GetPowerShellRunspace())
{
RunJobInPowershell(runspace, mapReduceJobDefinition);
}
}
public void UserCanSupplyStatusDirectoryForMapReduceJob()
{
var mapReduceJobDefinition = new AzureHDInsightMapReduceJobDefinition
{
JobName = "pi estimation jobDetails",
ClassName = "pi",
JarFile = TestConstants.WabsProtocolSchemeName + "container@hostname/examples.jar",
StatusFolder = "/myoutputfolder"
};
AzureHDInsightJob startedJob = TestJobStart(mapReduceJobDefinition);
Assert.AreEqual(startedJob.StatusDirectory, mapReduceJobDefinition.StatusFolder);
}
public void CanStartMapReduceJob()
{
var mapReduceJobDefinition = new AzureHDInsightMapReduceJobDefinition
{
JobName = "pi estimation jobDetails",
ClassName = "pi",
JarFile = TestConstants.WabsProtocolSchemeName + "container@hostname/examples.jar"
};
TestJobStart(mapReduceJobDefinition);
}
public void CanCreateNewMapReduceJob_WithoutJobName()
{
var mapReduceJobDefinition = new AzureHDInsightMapReduceJobDefinition
{
ClassName = "pi",
JarFile = TestConstants.WabsProtocolSchemeName + "container@hostname/examples.jar",
StatusFolder = "/myoutputfolder"
};
AzureHDInsightJob startedJob = TestJobStart(mapReduceJobDefinition);
Assert.AreEqual("pi", startedJob.Name);
}
