public virtual void PipeliningMapReduceJobExecution()
{
var mapReduceJobDefinition = new MapReduceJobCreateParameters
{
JobName = "pi estimation job",
ClassName = "pi",
JarFile = "/example/hadoop-examples.jar"
};
mapReduceJobDefinition.Arguments.Add("16");
mapReduceJobDefinition.Arguments.Add("10000");
using (IRunspace runspace = this.GetPowerShellRunspace())
{
ClusterDetails testCluster = GetHttpAccessEnabledCluster();
IPipelineResult results =
runspace.NewPipeline()
.AddCommand(CmdletConstants.NewAzureHDInsightMapReduceJobDefinition)
.WithParameter(CmdletConstants.JobName, mapReduceJobDefinition.JobName)
.WithParameter(CmdletConstants.JarFile, mapReduceJobDefinition.JarFile)
.WithParameter(CmdletConstants.ClassName, mapReduceJobDefinition.ClassName)
.WithParameter(CmdletConstants.Arguments, mapReduceJobDefinition.Arguments)
.AddCommand(CmdletConstants.StartAzureHDInsightJob)
.WithParameter(CmdletConstants.Cluster, testCluster.Name)
.AddCommand(CmdletConstants.WaitAzureHDInsightJob)
.WithParameter(CmdletConstants.WaitTimeoutInSeconds, 10)
.AddCommand(CmdletConstants.GetAzureHDInsightJobOutput)
.WithParameter(CmdletConstants.Cluster, testCluster.Name)
.Invoke();
Assert.IsNotNull(results.Results.ToEnumerable <string>());
Assert.IsTrue(results.Results.ToEnumerable <string>().Any(str => str == "3.142"));
}
}
public virtual void ICanCallThe_NewMapReduceJob_Then_Start_HDInsightJobsCmdlet()
{
var mapReduceJobDefinition = new MapReduceJobCreateParameters
{
JobName = "pi estimation jobDetails",
ClassName = "pi",
JarFile = "/example/hadoop-examples.jar"
};
mapReduceJobDefinition.Arguments.Add("16");
mapReduceJobDefinition.Arguments.Add("10000");
using (IRunspace runspace = this.GetPowerShellRunspace())
{
IPipelineResult results =
runspace.NewPipeline()
.AddCommand(CmdletConstants.NewAzureHDInsightMapReduceJobDefinition)
.WithParameter(CmdletConstants.JobName, mapReduceJobDefinition.JobName)
.WithParameter(CmdletConstants.JarFile, mapReduceJobDefinition.JarFile)
.WithParameter(CmdletConstants.ClassName, mapReduceJobDefinition.ClassName)
.WithParameter(CmdletConstants.Arguments, mapReduceJobDefinition.Arguments)
.Invoke();
Assert.AreEqual(1, results.Results.Count);
AzureHDInsightMapReduceJobDefinition mapReduceJobFromPowershell =
results.Results.ToEnumerable <AzureHDInsightMapReduceJobDefinition>().First();
RunJobInPowershell(runspace, mapReduceJobFromPowershell);
}
}
/// <summary>
/// Converts the Powershell object type to SDK object type.
/// </summary>
/// <returns>An SDK MapReduce object type.</returns>
internal MapReduceJobCreateParameters ToMapReduceJobCreateParameters()
{
var mapReduceJobDefinition = new MapReduceJobCreateParameters
{
ClassName = this.ClassName,
JarFile = this.JarFile,
JobName = this.JobName,
StatusFolder = this.StatusFolder
};
if (this.Arguments.IsNotNull())
{
mapReduceJobDefinition.Arguments.AddRange(this.Arguments);
}
if (this.Defines.IsNotNull())
{
mapReduceJobDefinition.Defines.AddRange(this.Defines);
}
if (this.Files.IsNotNull())
{
mapReduceJobDefinition.Files.AddRange(this.Files);
}
return(mapReduceJobDefinition);
}
/// <summary>
/// Converts the Powershell object type to SDK object type.
/// </summary>
/// <returns>An SDK MapReduce object type.</returns>
internal MapReduceJobCreateParameters ToMapReduceJobCreateParameters()
{
var mapReduceJobDefinition = new MapReduceJobCreateParameters
{
ClassName = this.ClassName,
JarFile = this.JarFile,
JobName = this.JobName,
StatusFolder = this.StatusFolder
};
if (this.Arguments.IsNotNull())
{
mapReduceJobDefinition.Arguments.AddRange(this.Arguments);
}
if (this.Defines.IsNotNull())
{
mapReduceJobDefinition.Defines.AddRange(this.Defines);
}
if (this.Files.IsNotNull())
{
mapReduceJobDefinition.Files.AddRange(this.Files);
}
return mapReduceJobDefinition;
}
public void ICanCallThe_New_HDInsightMapReduceJobDefinitionCmdlet()
{
var mapReduceJobDefinition = new MapReduceJobCreateParameters
{
JobName = "pi estimation jobDetails",
ClassName = "pi",
JarFile = TestConstants.WabsProtocolSchemeName + "container@hostname/examples.jar"
};
using (IRunspace runspace = this.GetPowerShellRunspace())
{
IPipelineResult results =
runspace.NewPipeline()
.AddCommand(CmdletConstants.NewAzureHDInsightMapReduceJobDefinition)
.WithParameter(CmdletConstants.JobName, mapReduceJobDefinition.JobName)
.WithParameter(CmdletConstants.JarFile, mapReduceJobDefinition.JarFile)
.WithParameter(CmdletConstants.ClassName, mapReduceJobDefinition.ClassName)
.Invoke();
Assert.AreEqual(1, results.Results.Count);
AzureHDInsightMapReduceJobDefinition mapReduceJobFromPowershell =
results.Results.ToEnumerable<AzureHDInsightMapReduceJobDefinition>().First();
Assert.AreEqual(mapReduceJobDefinition.JobName, mapReduceJobFromPowershell.JobName);
Assert.AreEqual(mapReduceJobDefinition.ClassName, mapReduceJobFromPowershell.ClassName);
Assert.AreEqual(mapReduceJobDefinition.JarFile, mapReduceJobFromPowershell.JarFile);
}
}
public virtual void ICanCallThe_NewMapReduceJob_Then_Start_HDInsightJobsCmdlet()
{
var mapReduceJobDefinition = new MapReduceJobCreateParameters
{
JobName = "pi estimation jobDetails",
ClassName = "pi",
JarFile = "/example/hadoop-examples.jar"
};
mapReduceJobDefinition.Arguments.Add("16");
mapReduceJobDefinition.Arguments.Add("10000");
using (IRunspace runspace = this.GetPowerShellRunspace())
{
IPipelineResult results =
runspace.NewPipeline()
.AddCommand(CmdletConstants.NewAzureHDInsightMapReduceJobDefinition)
.WithParameter(CmdletConstants.JobName, mapReduceJobDefinition.JobName)
.WithParameter(CmdletConstants.JarFile, mapReduceJobDefinition.JarFile)
.WithParameter(CmdletConstants.ClassName, mapReduceJobDefinition.ClassName)
.WithParameter(CmdletConstants.Arguments, mapReduceJobDefinition.Arguments)
.Invoke();
Assert.AreEqual(1, results.Results.Count);
AzureHDInsightMapReduceJobDefinition mapReduceJobFromPowershell =
results.Results.ToEnumerable<AzureHDInsightMapReduceJobDefinition>().First();
RunJobInPowershell(runspace, mapReduceJobFromPowershell);
}
}
public void CanSerializeValidJobRequest_Arguments_And_Defines()
{
var piJob = new MapReduceJobCreateParameters();
piJob.JarFile = "hadoop-examples.jar";
piJob.Arguments.Add("16");
piJob.Arguments.Add("10000");
piJob.Defines.Add("map.red.tasks", "1000");
piJob.Defines.Add("other.tasks", "1000");
var payloadConverter = new PayloadConverterBase();
var payload = payloadConverter.SerializeMapReduceRequest("hadoop", piJob);
foreach (var argument in piJob.Arguments)
{
Assert.IsTrue(payload.Contains(string.Format("{0}={1}", WebHCatConstants.Arg, argument)));
}
int defineCounter = 0;
foreach (var define in piJob.Defines)
{
defineCounter++;
Assert.IsTrue(payload.Contains(Uri.EscapeDataString(string.Format("{0}={1}", define.Key, define.Value))));
}
Assert.AreEqual(piJob.Defines.Count, defineCounter);
}
public void CanCreateNewMapReduceDefinition_WithArguments()
{
var mapReduceJobDefinition = new MapReduceJobCreateParameters
{
JobName = "pi estimation jobDetails",
ClassName = "pi",
JarFile = TestConstants.WabsProtocolSchemeName + "container@hostname/examples.jar"
};
mapReduceJobDefinition.Arguments.Add("16");
mapReduceJobDefinition.Arguments.Add("10000");
INewAzureHDInsightMapReduceJobDefinitionCommand newMapReduceJobDefinitionCommand =
ServiceLocator.Instance.Locate <IAzureHDInsightCommandFactory>().CreateNewMapReduceDefinition();
newMapReduceJobDefinitionCommand.JobName = mapReduceJobDefinition.JobName;
newMapReduceJobDefinitionCommand.JarFile = mapReduceJobDefinition.JarFile;
newMapReduceJobDefinitionCommand.ClassName = mapReduceJobDefinition.ClassName;
newMapReduceJobDefinitionCommand.Arguments = mapReduceJobDefinition.Arguments.ToArray();
newMapReduceJobDefinitionCommand.EndProcessing();
AzureHDInsightMapReduceJobDefinition mapReduceJobFromCommand = newMapReduceJobDefinitionCommand.Output.ElementAt(0);
Assert.AreEqual(mapReduceJobDefinition.JobName, mapReduceJobFromCommand.JobName);
Assert.AreEqual(mapReduceJobDefinition.ClassName, mapReduceJobFromCommand.ClassName);
Assert.AreEqual(mapReduceJobDefinition.JarFile, mapReduceJobFromCommand.JarFile);
foreach (string argument in mapReduceJobDefinition.Arguments)
{
Assert.IsTrue(
mapReduceJobFromCommand.Arguments.Any(arg => string.Equals(argument, arg)),
"Unable to find argument '{0}' in value returned from command",
argument);
}
}
public void CanCreateNewMapReduceDefinition_WithArguments()
{
var mapReduceJobDefinition = new MapReduceJobCreateParameters
{
JobName = "pi estimation jobDetails",
ClassName = "pi",
JarFile = TestConstants.WabsProtocolSchemeName + "container@hostname/examples.jar"
};
mapReduceJobDefinition.Arguments.Add("16");
mapReduceJobDefinition.Arguments.Add("10000");
INewAzureHDInsightMapReduceJobDefinitionCommand newMapReduceJobDefinitionCommand =
ServiceLocator.Instance.Locate<IAzureHDInsightCommandFactory>().CreateNewMapReduceDefinition();
newMapReduceJobDefinitionCommand.JobName = mapReduceJobDefinition.JobName;
newMapReduceJobDefinitionCommand.JarFile = mapReduceJobDefinition.JarFile;
newMapReduceJobDefinitionCommand.ClassName = mapReduceJobDefinition.ClassName;
newMapReduceJobDefinitionCommand.Arguments = mapReduceJobDefinition.Arguments.ToArray();
newMapReduceJobDefinitionCommand.EndProcessing();
AzureHDInsightMapReduceJobDefinition mapReduceJobFromCommand = newMapReduceJobDefinitionCommand.Output.ElementAt(0);
Assert.AreEqual(mapReduceJobDefinition.JobName, mapReduceJobFromCommand.JobName);
Assert.AreEqual(mapReduceJobDefinition.ClassName, mapReduceJobFromCommand.ClassName);
Assert.AreEqual(mapReduceJobDefinition.JarFile, mapReduceJobFromCommand.JarFile);
foreach (string argument in mapReduceJobDefinition.Arguments)
{
Assert.IsTrue(
mapReduceJobFromCommand.Arguments.Any(arg => string.Equals(argument, arg)),
"Unable to find argument '{0}' in value returned from command",
argument);
}
}
public void ICanCallThe_New_HDInsightMapReduceJobDefinitionCmdlet()
{
var mapReduceJobDefinition = new MapReduceJobCreateParameters
{
JobName = "pi estimation jobDetails",
ClassName = "pi",
JarFile = TestConstants.WabsProtocolSchemeName + "container@hostname/examples.jar",
};
mapReduceJobDefinition.LibJars.Add("some.jarfile.jar");
using (IRunspace runspace = this.GetPowerShellRunspace())
{
IPipelineResult results =
runspace.NewPipeline()
.AddCommand(CmdletConstants.NewAzureHDInsightMapReduceJobDefinition)
.WithParameter(CmdletConstants.JobName, mapReduceJobDefinition.JobName)
.WithParameter(CmdletConstants.JarFile, mapReduceJobDefinition.JarFile)
.WithParameter(CmdletConstants.ClassName, mapReduceJobDefinition.ClassName)
.WithParameter(CmdletConstants.LibJars, mapReduceJobDefinition.LibJars)
.Invoke();
Assert.AreEqual(1, results.Results.Count);
AzureHDInsightMapReduceJobDefinition mapReduceJobFromPowershell =
results.Results.ToEnumerable <AzureHDInsightMapReduceJobDefinition>().First();
Assert.AreEqual(mapReduceJobDefinition.JobName, mapReduceJobFromPowershell.JobName);
Assert.AreEqual(mapReduceJobDefinition.ClassName, mapReduceJobFromPowershell.ClassName);
Assert.AreEqual(mapReduceJobDefinition.JarFile, mapReduceJobFromPowershell.JarFile);
Assert.AreEqual(mapReduceJobDefinition.LibJars.Count, mapReduceJobFromPowershell.LibJars.Count);
Assert.AreEqual(mapReduceJobDefinition.LibJars.First(), mapReduceJobFromPowershell.LibJars.First());
}
}
private static async Task <JobCreationResults> SubmitMapReduceJob(
AzureHDInsightMapReduceJobDefinition azureMapReduceJobDefinition, IJobSubmissionClient client)
{
MapReduceJobCreateParameters mapReduceJobDefinition = azureMapReduceJobDefinition.ToMapReduceJobCreateParameters();
var jobCreationResults = await client.CreateMapReduceJobAsync(mapReduceJobDefinition);
return(jobCreationResults);
}
/// <inheritdoc />
public async Task<JobCreationResults> SubmitMapReduceJob(MapReduceJobCreateParameters details)
{
//NEIN: Any code modification here should add unit tests for this class
var converter = new PayloadConverter();
var payload = converter.SerializeMapReduceRequest(this.credentials.UserName, details);
var client = ServiceLocator.Instance.Locate<IHadoopRemoteJobSubmissionRestClientFactory>().Create(this.credentials, this.context, this.ignoreSslErrors, this.GetUserAgentString());
var result = await client.SubmitMapReduceJob(payload);
return new JobCreationResults() { JobId = converter.DeserializeJobSubmissionResponse(result.Content) };
}
public void ServiceHost_JobSubmissionRecieved(object sender, JobSubmissionMessage e)
{
Trace.WriteLine("JobSubmissionRecieved Recieved User Id : " + e.idUsuario, "Warning");
try
{
// Obtener la cuenta de almacenamiento
// Para actualizar metadatos
CloudStorageAccount storageAccount = CloudStorageAccount.Parse(
CloudConfigurationManager.GetSetting("StorageConnectionString"));
CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient();
CloudBlobContainer container = blobClient.GetContainerReference(VariablesConfiguracion.containerName);
// Obtencion de variables para la conxion con el cluster
string subscriptionID = VariablesConfiguracion.subscriptionID;
string certFriendlyName = VariablesConfiguracion.certFriendlyName;
string clusterName = VariablesConfiguracion.clusterName;
// Definicion de la tarea MapReduce
MapReduceJobCreateParameters mrJobDefinition = new MapReduceJobCreateParameters()
{
JarFile = "wasb:///CienciaCelularMR.jar",
ClassName = "Main",
StatusFolder = "wasb:///scicluster/test/status-" + e.idUsuario + "." + e.subIdUsuario,
};
mrJobDefinition.Arguments.Add("wasb:///" + e.nomEntrada);
mrJobDefinition.Arguments.Add("wasb:///scicluster/test/output-" + e.idUsuario + "." + e.subIdUsuario);
// Obtener el objeto certificate
X509Store store = new X509Store();
store.Open(OpenFlags.ReadOnly);
X509Certificate2 cert = FindCertificate(StoreLocation.CurrentUser, StoreName.My, X509FindType.FindByThumbprint, VariablesConfiguracion.thumbprint);
JobSubmissionCertificateCredential creds = new JobSubmissionCertificateCredential(new Guid(subscriptionID), cert, clusterName);
// Se crea un cliente Hadoop para conectarse con HDInsight
var jobClient = JobSubmissionClientFactory.Connect(creds);
//Actualizacion de metadatos
CloudBlockBlob infoSimulation = container.GetBlockBlobReference(VariablesConfiguracion.infoSimulation + "-" + e.idUsuario);
infoSimulation.UploadText(VariablesConfiguracion.JOB_STARTING);
// Se lanza la ejecucion de los jobs MapReduce
JobCreationResults mrJobResults = jobClient.CreateMapReduceJob(mrJobDefinition);
// Esperar hasta que finalice la ejecucion
WaitForJobCompletion(mrJobResults, jobClient, e.idUsuario, e.subIdUsuario);
}
catch (Exception ex)
{
Trace.TraceError(ex.Message);
throw;
}
}
public Task <JobCreationResults> SubmitMapReduceJob(MapReduceJobCreateParameters details)
{
this.SubmitMapReduceJobCalled = true;
var job = new JobCreationResults()
{
JobId = JobId
};
return(Task.Run(() => job));
}
public void GivenIHaveAMapReduceJobRequestObject()
{
var request = new MapReduceJobCreateParameters();
request.ClassName = string.Empty;
request.JarFile = string.Empty;
request.JobName = string.Empty;
request.StatusFolder = string.Empty;
this.transferObject = request;
}
/// <summary>
/// Creates the payload for a MapReduce request.
/// </summary>
/// <param name="userName">
/// The user name.
/// </param>
/// <param name="details">
/// The details.
/// </param>
/// <returns>
/// A string that represents the payload for the request.
/// </returns>
public string SerializeMapReduceRequest(string userName, MapReduceJobCreateParameters details)
{
details.ArgumentNotNull("details");
var values = new List<KeyValuePair<string, string>>();
values.AddRange(this.SerializeJobRequest(userName, details, details.JobName, details.Arguments, details.Defines));
values.Add(new KeyValuePair<string, string>(WebHCatConstants.Jar, details.JarFile));
values.Add(new KeyValuePair<string, string>(WebHCatConstants.Class, details.ClassName));
var retval = this.ConvertItems(values.Where(kvp => kvp.Value != null));
return retval;
}
public Task <JobCreationResults> CreateMapReduceJobAsync(MapReduceJobCreateParameters mapReduceJobCreateParameters)
{
if (mapReduceJobCreateParameters.JobName == "1456577")
{
throw new HttpLayerException(HttpStatusCode.BadRequest, "{ \"error\": \"File /example/files/WordCount.jar does not exist.\"}");
}
mapReduceJobCreateParameters.JarFile.ArgumentNotNullOrEmpty("JarFile");
mapReduceJobCreateParameters.ClassName.ArgumentNotNullOrEmpty("ClassName");
return(TaskEx2.FromResult(this.CreateJobSuccessResult(mapReduceJobCreateParameters, mapReduceJobCreateParameters.JobName)));
}
/// <summary>
/// Creates the payload for a MapReduce request.
/// </summary>
/// <param name="userName">
/// The user name.
/// </param>
/// <param name="details">
/// The details.
/// </param>
/// <returns>
/// A string that represents the payload for the request.
/// </returns>
public string SerializeMapReduceRequest(string userName, MapReduceJobCreateParameters details)
{
details.ArgumentNotNull("details");
var values = new List <KeyValuePair <string, string> >();
values.AddRange(this.SerializeJobRequest(userName, details, details.JobName, details.Arguments, details.Defines));
values.Add(new KeyValuePair <string, string>(WebHCatConstants.Jar, details.JarFile));
values.Add(new KeyValuePair <string, string>(WebHCatConstants.Class, details.ClassName));
var retval = this.ConvertItems(values.Where(kvp => kvp.Value != null));
return(retval);
}
public void CanSerializeValidJobRequest_JobName()
{
var mapReduceJob = new MapReduceJobCreateParameters()
{
JobName = "pi estimation jobDetails"
};
var payloadConverter = new PayloadConverterBase();
var payload = payloadConverter.SerializeMapReduceRequest("hadoop", mapReduceJob);
Assert.IsTrue(payload.Contains(Uri.EscapeDataString(string.Format("{0}={1}", WebHCatConstants.DefineJobName, mapReduceJob.JobName))));
}
/// <inheritdoc />
public async Task <JobCreationResults> SubmitMapReduceJob(MapReduceJobCreateParameters details)
{
//NEIN: Any code modification here should add unit tests for this class
var converter = new PayloadConverter();
var payload = converter.SerializeMapReduceRequest(this.credentials.UserName, details);
var client = ServiceLocator.Instance.Locate <IHadoopRemoteJobSubmissionRestClientFactory>().Create(this.credentials, this.context, this.ignoreSslErrors, this.GetUserAgentString());
var result = await client.SubmitMapReduceJob(payload);
return(new JobCreationResults()
{
JobId = converter.DeserializeJobSubmissionResponse(result.Content)
});
}
public async Task<JobCreationResults> SubmitMapReduceJob(MapReduceJobCreateParameters details)
{
var relative = new Uri(
HadoopRemoteRestConstants.MapReduceJar + "?" +
HadoopRemoteRestConstants.UserName + "=" +
_connectionConfig.UserName.EscapeDataString(),
UriKind.Relative);
var requestContent = _converter.SerializeMapReduceRequest(_connectionConfig.UserName, details);
var result = await MakeAsyncJobSubmissionRequest(relative, requestContent);
return new JobCreationResults { JobId = _converter.DeserializeJobSubmissionResponse(result) };
}
/// <inheritdoc/>
public async Task <JobCreationResults> CreateMapReduceJobAsync(MapReduceJobCreateParameters mapReduceJobCreateParameters)
{
var pocoClient = this.GetPocoClient();
try
{
return(await pocoClient.SubmitMapReduceJob(mapReduceJobCreateParameters));
}
catch (UnauthorizedAccessException)
{
pocoClient = this.GetPocoClient(true);
}
return(await pocoClient.SubmitMapReduceJob(mapReduceJobCreateParameters));
}
public void CanSerializeValidMapReduceJobRequest()
{
var mapReduceJob = new MapReduceJobCreateParameters()
{
JarFile = "/example/hadoop-examples.jar",
ClassName = "pi"
};
var payloadConverter = new PayloadConverterBase();
var payload = payloadConverter.SerializeMapReduceRequest("hadoop", mapReduceJob);
Assert.IsTrue(payload.Contains(string.Format("{0}={1}", WebHCatConstants.Jar, Uri.EscapeDataString(mapReduceJob.JarFile))));
Assert.IsTrue(payload.Contains(string.Format("{0}={1}", WebHCatConstants.Class, Uri.EscapeDataString(mapReduceJob.ClassName))));
}
public void CanSerializeValidJobRequestWithCallback()
{
var mapReduceJob = new MapReduceJobCreateParameters()
{
JobName = "pi estimation jobDetails",
Callback = "http://someball.com/$jobid/notvalid"
};
var payloadConverter = new PayloadConverterBase();
var payload = payloadConverter.SerializeMapReduceRequest("hadoop", mapReduceJob);
Assert.IsTrue(payload.Contains(Uri.EscapeDataString(string.Format("{0}={1}", WebHCatConstants.DefineJobName, mapReduceJob.JobName))));
Assert.IsTrue(payload.Contains(string.Format("{0}={1}", Uri.EscapeDataString(WebHCatConstants.Callback), Uri.EscapeDataString(mapReduceJob.Callback))));
}
public async Task <JobCreationResults> SubmitMapReduceJob(MapReduceJobCreateParameters details)
{
var relative = new Uri(
HadoopRemoteRestConstants.MapReduceJar + "?" +
HadoopRemoteRestConstants.UserName + "=" +
_connectionConfig.UserName.EscapeDataString(),
UriKind.Relative);
var requestContent = _converter.SerializeMapReduceRequest(_connectionConfig.UserName, details);
var result = await MakeAsyncJobSubmissionRequest(relative, requestContent);
return(new JobCreationResults {
JobId = _converter.DeserializeJobSubmissionResponse(result)
});
}
//Run Sample Map Reduce Job
public static void DoMapReduce()
{
// Define the MapReduce job
MapReduceJobCreateParameters mrJobDefinition = new MapReduceJobCreateParameters()
{
JarFile = "wasb:///example/jars/hadoop-examples.jar",
ClassName = "wordcount"
};
mrJobDefinition.Arguments.Add("wasb:///example/data/gutenberg/davinci.txt");
mrJobDefinition.Arguments.Add("wasb:///example/data/WordCountOutput");
//Get certificate
var store = new X509Store();
store.Open(OpenFlags.ReadOnly);
var cert = store.Certificates.Cast <X509Certificate2>().First(item => item.Thumbprint == Constants.thumbprint);
var creds = new JobSubmissionCertificateCredential(Constants.subscriptionId, cert, Constants.clusterName);
// Create a hadoop client to connect to HDInsight
var jobClient = JobSubmissionClientFactory.Connect(creds);
// Run the MapReduce job
JobCreationResults mrJobResults = jobClient.CreateMapReduceJob(mrJobDefinition);
Console.Write("Executing WordCount MapReduce Job.");
// Wait for the job to complete
WaitForJobCompletion(mrJobResults, jobClient);
// Print the MapReduce job output
Stream stream = new MemoryStream();
CloudStorageAccount storageAccount = CloudStorageAccount.Parse("DefaultEndpointsProtocol=https;AccountName=" + Constants.storageAccount + ";AccountKey=" + Constants.storageAccountKey);
CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient();
CloudBlobContainer blobContainer = blobClient.GetContainerReference(Constants.container);
CloudBlockBlob blockBlob = blobContainer.GetBlockBlobReference("example/data/WordCountOutput/part-r-00000");
blockBlob.DownloadToStream(stream);
stream.Position = 0;
StreamReader reader = new StreamReader(stream);
Console.Write("Done..Word counts are:\n");
Console.WriteLine(reader.ReadToEnd());
}
public void CanCreateNewMapReduceJob_StartJob_StopJob()
{
var mapReduceJobDefinition = new MapReduceJobCreateParameters
{
JobName = "pi estimation job",
ClassName = "pi",
JarFile = TestConstants.WabsProtocolSchemeName + "container@hostname/examples.jar"
};
INewAzureHDInsightMapReduceJobDefinitionCommand newMapReduceJobDefinitionCommand =
ServiceLocator.Instance.Locate<IAzureHDInsightCommandFactory>().CreateNewMapReduceDefinition();
newMapReduceJobDefinitionCommand.JobName = mapReduceJobDefinition.JobName;
newMapReduceJobDefinitionCommand.JarFile = mapReduceJobDefinition.JarFile;
newMapReduceJobDefinitionCommand.ClassName = mapReduceJobDefinition.ClassName;
newMapReduceJobDefinitionCommand.EndProcessing();
AzureHDInsightMapReduceJobDefinition mapReduceJobFromCommand = newMapReduceJobDefinitionCommand.Output.ElementAt(0);
TestJobLifecycle(mapReduceJobFromCommand);
}
public MapReduceJobCreateParameters DeserializeMapReduceJobCreationDetails(string content)
{
var request = this.DeserializePayload(content);
var result = new MapReduceJobCreateParameters()
{
ClassName = request.ApplicationName,
JarFile = request.JarFile,
JobName = request.JobName,
StatusFolder = request.OutputStorageLocation,
};
result.Arguments.AddRange(request.Arguments);
foreach (var jobRequestParameter in request.Parameters)
{
result.Defines.Add(jobRequestParameter.Key, jobRequestParameter.Value.ToString());
}
this.SetStandardProperties(request, result);
return(result);
}
public void CanCreateNewMapReduceJob_StartJob()
{
var mapReduceJobDefinition = new MapReduceJobCreateParameters
{
JobName = "pi estimation jobDetails",
ClassName = "pi",
JarFile = TestConstants.WabsProtocolSchemeName + "container@hostname/examples.jar"
};
INewAzureHDInsightMapReduceJobDefinitionCommand newMapReduceJobDefinitionCommand =
ServiceLocator.Instance.Locate <IAzureHDInsightCommandFactory>().CreateNewMapReduceDefinition();
newMapReduceJobDefinitionCommand.JobName = mapReduceJobDefinition.JobName;
newMapReduceJobDefinitionCommand.JarFile = mapReduceJobDefinition.JarFile;
newMapReduceJobDefinitionCommand.ClassName = mapReduceJobDefinition.ClassName;
newMapReduceJobDefinitionCommand.EndProcessing();
AzureHDInsightMapReduceJobDefinition mapReduceJobFromCommand = newMapReduceJobDefinitionCommand.Output.ElementAt(0);
TestJobStart(mapReduceJobFromCommand);
}
public void CanCreateNewMapReduceDefinition_WithParameters()
{
var mapReduceJobDefinition = new MapReduceJobCreateParameters
{
JobName = "pi estimation jobDetails",
ClassName = "pi",
JarFile = TestConstants.WabsProtocolSchemeName + "container@hostname/examples.jar"
};
mapReduceJobDefinition.Defines.Add("map.input.tasks", "1000");
mapReduceJobDefinition.Defines.Add("map.input.reducers", "1000");
INewAzureHDInsightMapReduceJobDefinitionCommand newMapReduceJobDefinitionCommand =
ServiceLocator.Instance.Locate <IAzureHDInsightCommandFactory>().CreateNewMapReduceDefinition();
newMapReduceJobDefinitionCommand.JobName = mapReduceJobDefinition.JobName;
newMapReduceJobDefinitionCommand.JarFile = mapReduceJobDefinition.JarFile;
newMapReduceJobDefinitionCommand.ClassName = mapReduceJobDefinition.ClassName;
foreach (var define in mapReduceJobDefinition.Defines)
{
newMapReduceJobDefinitionCommand.Defines.Add(define.Key, define.Value);
}
newMapReduceJobDefinitionCommand.EndProcessing();
AzureHDInsightMapReduceJobDefinition mapReduceJobFromCommand = newMapReduceJobDefinitionCommand.Output.ElementAt(0);
Assert.AreEqual(mapReduceJobDefinition.JobName, mapReduceJobFromCommand.JobName);
Assert.AreEqual(mapReduceJobDefinition.ClassName, mapReduceJobFromCommand.ClassName);
Assert.AreEqual(mapReduceJobDefinition.JarFile, mapReduceJobFromCommand.JarFile);
foreach (var parameter in mapReduceJobDefinition.Defines)
{
Assert.IsTrue(
mapReduceJobFromCommand.Defines.Any(arg => string.Equals(parameter.Key, arg.Key) && string.Equals(parameter.Value, arg.Value)),
"Unable to find parameter '{0}' in value returned from command",
parameter.Key);
}
}
public void ICanCallThe_New_HDInsightMapReduceJobDefinitionCmdlet_WithParameters()
{
var mapReduceJobDefinition = new MapReduceJobCreateParameters
{
JobName = "pi estimation jobDetails",
ClassName = "pi",
JarFile = TestConstants.WabsProtocolSchemeName + "container@hostname/examples.jar"
};
mapReduceJobDefinition.Defines.Add("map.input.tasks", "1000");
mapReduceJobDefinition.Defines.Add("map.input.reducers", "1000");
using (IRunspace runspace = this.GetPowerShellRunspace())
{
IPipelineResult results =
runspace.NewPipeline()
.AddCommand(CmdletConstants.NewAzureHDInsightMapReduceJobDefinition)
.WithParameter(CmdletConstants.JobName, mapReduceJobDefinition.JobName)
.WithParameter(CmdletConstants.JarFile, mapReduceJobDefinition.JarFile)
.WithParameter(CmdletConstants.ClassName, mapReduceJobDefinition.ClassName)
.WithParameter(CmdletConstants.Parameters, mapReduceJobDefinition.Defines)
.Invoke();
Assert.AreEqual(1, results.Results.Count);
AzureHDInsightMapReduceJobDefinition mapReduceJobFromPowershell =
results.Results.ToEnumerable <AzureHDInsightMapReduceJobDefinition>().First();
Assert.AreEqual(mapReduceJobDefinition.JobName, mapReduceJobFromPowershell.JobName);
Assert.AreEqual(mapReduceJobDefinition.ClassName, mapReduceJobFromPowershell.ClassName);
Assert.AreEqual(mapReduceJobDefinition.JarFile, mapReduceJobFromPowershell.JarFile);
foreach (var parameter in mapReduceJobDefinition.Defines)
{
Assert.IsTrue(
mapReduceJobFromPowershell.Defines.Any(
arg => string.Equals(parameter.Key, arg.Key) && string.Equals(parameter.Value, arg.Value)),
"Unable to find parameter '{0}' in value returned from powershell",
parameter.Key);
}
}
}
public void CanSerializeValidJobRequest_Defines()
{
var mapReduceJob = new MapReduceJobCreateParameters();
mapReduceJob.JobName = "Define counter test";
mapReduceJob.Defines.Add(new KeyValuePair <string, string>("map.input.tasks", "1000"));
mapReduceJob.Defines.Add(new KeyValuePair <string, string>("map.input.mappers", "6"));
mapReduceJob.Defines.Add(new KeyValuePair <string, string>("map.input.reducers", "16"));
var payloadConverter = new PayloadConverterBase();
var payload = payloadConverter.SerializeMapReduceRequest("hadoop", mapReduceJob);
int defineCounter = 0;
foreach (var define in mapReduceJob.Defines)
{
defineCounter++;
Assert.IsTrue(payload.Contains(Uri.EscapeDataString(string.Format("{0}={1}", define.Key, define.Value))));
}
Assert.AreEqual(mapReduceJob.Defines.Count, defineCounter);
}
public void ICanCallThe_New_HDInsightMapReduceJobDefinitionCmdlet_WithArguments()
{
var mapReduceJobDefinition = new MapReduceJobCreateParameters
{
JobName = "pi estimation jobDetails",
ClassName = "pi",
JarFile = TestConstants.WabsProtocolSchemeName + "container@hostname/examples.jar"
};
mapReduceJobDefinition.Arguments.Add("16");
mapReduceJobDefinition.Arguments.Add("10000");
using (IRunspace runspace = this.GetPowerShellRunspace())
{
IPipelineResult results =
runspace.NewPipeline()
.AddCommand(CmdletConstants.NewAzureHDInsightMapReduceJobDefinition)
.WithParameter(CmdletConstants.JobName, mapReduceJobDefinition.JobName)
.WithParameter(CmdletConstants.JarFile, mapReduceJobDefinition.JarFile)
.WithParameter(CmdletConstants.ClassName, mapReduceJobDefinition.ClassName)
.WithParameter(CmdletConstants.Arguments, mapReduceJobDefinition.Arguments)
.Invoke();
Assert.AreEqual(1, results.Results.Count);
AzureHDInsightMapReduceJobDefinition mapReduceJobFromPowershell =
results.Results.ToEnumerable<AzureHDInsightMapReduceJobDefinition>().First();
Assert.AreEqual(mapReduceJobDefinition.JobName, mapReduceJobFromPowershell.JobName);
Assert.AreEqual(mapReduceJobDefinition.ClassName, mapReduceJobFromPowershell.ClassName);
Assert.AreEqual(mapReduceJobDefinition.JarFile, mapReduceJobFromPowershell.JarFile);
foreach (string argument in mapReduceJobDefinition.Arguments)
{
Assert.IsTrue(
mapReduceJobFromPowershell.Arguments.Any(arg => string.Equals(argument, arg)),
"Unable to find argument '{0}' in value returned from powershell",
argument);
}
}
}
public Task<JobCreationResults> CreateMapReduceJobAsync(MapReduceJobCreateParameters mapReduceJobCreateParameters)
{
if (mapReduceJobCreateParameters.JobName == "1456577")
{
throw new HttpLayerException(HttpStatusCode.BadRequest, "{ \"error\": \"File /example/files/WordCount.jar does not exist.\"}");
}
mapReduceJobCreateParameters.JarFile.ArgumentNotNullOrEmpty("JarFile");
mapReduceJobCreateParameters.ClassName.ArgumentNotNullOrEmpty("ClassName");
return TaskEx2.FromResult(this.CreateJobSuccessResult(mapReduceJobCreateParameters, mapReduceJobCreateParameters.JobName));
}
public void ICanNotSubmitAJobWithTheIncorectCredintials()
{
IHDInsightCertificateCredential hdInsightCredentials = IntegrationTestBase.GetValidCredentials();
var client = ServiceLocator.Instance.Locate <IHDInsightClientFactory>().Create(new HDInsightCertificateCredential(hdInsightCredentials.SubscriptionId, hdInsightCredentials.Certificate));
var manager = ServiceLocator.Instance.Locate <IHDInsightManagementPocoClientFactory>();
var pocoClient = manager.Create(hdInsightCredentials, GetAbstractionContext(), false);
var clusterDetails = GetRandomCluster();
client.CreateCluster(clusterDetails);
try
{
ClusterDetails cluster = pocoClient.ListContainer(clusterDetails.Name).WaitForResult();
BasicAuthCredential hadoopCredentials = new BasicAuthCredential()
{
Server = GatewayUriResolver.GetGatewayUri(cluster.ConnectionUrl),
UserName = clusterDetails.UserName,
Password = clusterDetails.Password
};
var hadoopClient = JobSubmissionClientFactory.Connect(hadoopCredentials);
var mapReduceJob = new MapReduceJobCreateParameters()
{
ClassName = "pi",
JobName = "pi estimation jobDetails",
JarFile = "/example/hadoop-examples.jar",
StatusFolder = "/piresults"
};
mapReduceJob.Arguments.Add("16");
mapReduceJob.Arguments.Add("10000");
var jobCreationDetails = hadoopClient.CreateMapReduceJob(mapReduceJob);
var id = pocoClient.DisableHttp(clusterDetails.Name, clusterDetails.Location).WaitForResult();
while (!pocoClient.IsComplete(cluster.Name, cluster.Location, id).WaitForResult())
{
Thread.Sleep(500);
}
// now add a user
string userName = "******";
string password = GetRandomValidPassword();
id = pocoClient.EnableHttp(clusterDetails.Name, clusterDetails.Location, userName, password).WaitForResult();
while (!pocoClient.IsComplete(cluster.Name, cluster.Location, id).WaitForResult())
{
Thread.Sleep(500);
}
jobCreationDetails = hadoopClient.CreateMapReduceJob(mapReduceJob);
Assert.Fail("This test expected an exception but did not receive one.");
}
catch (UnauthorizedAccessException ex)
{
Help.DoNothing(ex);
}
finally
{
// delete the cluster
client.DeleteCluster(clusterDetails.Name);
}
}
public JobCreationResults CreateMapReduceJob(MapReduceJobCreateParameters mapReduceJobCreateParameters)
{
return this.CreateMapReduceJobAsync(mapReduceJobCreateParameters).WaitForResult();
}
public void TheValueOfTheSerializedOutputShouldBeEquivalentWithTheOrignal()
{
var clientConverter = new JobPayloadConverter();
var serverConverter = new JobPayloadServerConverter();
JobList asList = this.transferObject.As <JobList>();
if (asList.IsNotNull())
{
JobList actual = clientConverter.DeserializeJobList(this.serializedForm);
Assert.AreEqual(asList.ErrorCode, actual.ErrorCode);
Assert.AreEqual(asList.HttpStatusCode, actual.HttpStatusCode);
Assert.IsTrue(asList.Jobs.Select(j => j.JobId).SequenceEqual(actual.Jobs.Select(j => j.JobId)));
return;
}
var asJob = this.transferObject.As <Hadoop.Client.JobDetails>();
if (asJob.IsNotNull())
{
var actual = clientConverter.DeserializeJobDetails(this.serializedForm, asJob.JobId);
Assert.AreEqual(asJob.ErrorCode, actual.ErrorCode);
Assert.AreEqual(asJob.HttpStatusCode, actual.HttpStatusCode);
Assert.AreEqual(asJob.ErrorOutputPath, actual.ErrorOutputPath);
Assert.AreEqual(asJob.ExitCode, actual.ExitCode);
Assert.AreEqual(asJob.JobId, actual.JobId);
Assert.AreEqual(asJob.LogicalOutputPath, actual.LogicalOutputPath);
Assert.AreEqual(asJob.Name, actual.Name);
Assert.AreEqual(new Uri(asJob.PhysicalOutputPath), new Uri(actual.PhysicalOutputPath));
Assert.AreEqual(asJob.Query, actual.Query);
Assert.AreEqual(asJob.StatusCode, actual.StatusCode);
Assert.AreEqual(asJob.SubmissionTime, actual.SubmissionTime);
Assert.AreEqual(asJob.Callback, actual.Callback);
return;
}
var asMapReduce = this.transferObject.As <MapReduceJobCreateParameters>();
if (asMapReduce.IsNotNull())
{
MapReduceJobCreateParameters actual = serverConverter.DeserializeMapReduceJobCreationDetails(this.serializedForm);
Assert.AreEqual(asMapReduce.ClassName, actual.ClassName);
Assert.IsTrue(asMapReduce.Arguments.SequenceEqual(actual.Arguments));
Assert.AreEqual(asMapReduce.JarFile, actual.JarFile);
Assert.AreEqual(asMapReduce.JobName, actual.JobName);
Assert.AreEqual(asMapReduce.StatusFolder, actual.StatusFolder);
Assert.IsTrue(asMapReduce.Defines.SequenceEqual(actual.Defines));
Assert.IsTrue(asMapReduce.Files.SequenceEqual(actual.Files));
return;
}
var asHive = this.transferObject.As <HiveJobCreateParameters>();
if (asHive.IsNotNull())
{
HiveJobCreateParameters actual = serverConverter.DeserializeHiveJobCreationDetails(this.serializedForm);
Assert.AreEqual(asHive.JobName, actual.JobName);
Assert.AreEqual(asHive.StatusFolder, actual.StatusFolder);
Assert.IsTrue(asHive.Defines.SequenceEqual(actual.Defines));
Assert.AreEqual(asHive.Query, actual.Query);
Assert.IsTrue(asHive.Files.SequenceEqual(actual.Files));
return;
}
Assert.Fail("Unable to recognize the object type.");
}
public void CanCreateNewMapReduceDefinition_WithParameters()
{
var mapReduceJobDefinition = new MapReduceJobCreateParameters
{
JobName = "pi estimation jobDetails",
ClassName = "pi",
JarFile = TestConstants.WabsProtocolSchemeName + "container@hostname/examples.jar"
};
mapReduceJobDefinition.Defines.Add("map.input.tasks", "1000");
mapReduceJobDefinition.Defines.Add("map.input.reducers", "1000");
INewAzureHDInsightMapReduceJobDefinitionCommand newMapReduceJobDefinitionCommand =
ServiceLocator.Instance.Locate<IAzureHDInsightCommandFactory>().CreateNewMapReduceDefinition();
newMapReduceJobDefinitionCommand.JobName = mapReduceJobDefinition.JobName;
newMapReduceJobDefinitionCommand.JarFile = mapReduceJobDefinition.JarFile;
newMapReduceJobDefinitionCommand.ClassName = mapReduceJobDefinition.ClassName;
foreach (var define in mapReduceJobDefinition.Defines)
{
newMapReduceJobDefinitionCommand.Defines.Add(define.Key, define.Value);
}
newMapReduceJobDefinitionCommand.EndProcessing();
AzureHDInsightMapReduceJobDefinition mapReduceJobFromCommand = newMapReduceJobDefinitionCommand.Output.ElementAt(0);
Assert.AreEqual(mapReduceJobDefinition.JobName, mapReduceJobFromCommand.JobName);
Assert.AreEqual(mapReduceJobDefinition.ClassName, mapReduceJobFromCommand.ClassName);
Assert.AreEqual(mapReduceJobDefinition.JarFile, mapReduceJobFromCommand.JarFile);
foreach (var parameter in mapReduceJobDefinition.Defines)
{
Assert.IsTrue(
mapReduceJobFromCommand.Defines.Any(arg => string.Equals(parameter.Key, arg.Key) && string.Equals(parameter.Value, arg.Value)),
"Unable to find parameter '{0}' in value returned from command",
parameter.Key);
}
}
public JobCreationResults CreateMapReduceJob(MapReduceJobCreateParameters mapReduceJobCreateParameters)
{
return(this.CreateMapReduceJobAsync(mapReduceJobCreateParameters).WaitForResult());
}
/// <inheritdoc />
public async Task <JobCreationResults> SubmitMapReduceJob(MapReduceJobCreateParameters details)
{
var remoteClient = ServiceLocator.Instance.Locate <IRemoteHadoopJobSubmissionPocoClientFactory>().Create(this.remoteCreds, this.context, this.ignoreSslErrors, this.GetUserAgentString());
return(await remoteClient.SubmitMapReduceJob(details));
}
