//Run Hive Job
public static void DoHiveOperations()
{
HiveJobCreateParameters hiveJobDefinition = new HiveJobCreateParameters()
{
JobName = "Show tables job",
StatusFolder = "/TableListFolder",
Query = "show tables;"
};
var store = new X509Store();
store.Open(OpenFlags.ReadOnly);
var cert = store.Certificates.Cast <X509Certificate2>().First(item => item.Thumbprint == Constants.thumbprint);
var creds = new JobSubmissionCertificateCredential(Constants.subscriptionId, cert, Constants.clusterName);
var jobClient = JobSubmissionClientFactory.Connect(creds);
JobCreationResults jobResults = jobClient.CreateHiveJob(hiveJobDefinition);
Console.Write("Executing Hive Job.");
// Wait for the job to complete
WaitForJobCompletion(jobResults, jobClient);
// Print the Hive job output
System.IO.Stream stream = jobClient.GetJobOutput(jobResults.JobId);
System.IO.StreamReader reader = new System.IO.StreamReader(stream);
Console.Write("Done..List of Tables are:\n");
Console.WriteLine(reader.ReadToEnd());
}
public static IJobSubmissionClientCredential GetJobSubmissionClientCredentials(this IAzureHDInsightJobCommandCredentialsBase command, WindowsAzureSubscription currentSubscription, string cluster)
{
IJobSubmissionClientCredential clientCredential = null;
if (command.Credential != null)
{
clientCredential = new BasicAuthCredential
{
Server = GatewayUriResolver.GetGatewayUri(cluster),
UserName = command.Credential.UserName,
Password = command.Credential.GetCleartextPassword()
};
}
else if (currentSubscription.IsNotNull())
{
var subscriptionCredentials = GetSubscriptionCredentials(command, currentSubscription);
var asCertificateCredentials = subscriptionCredentials as HDInsightCertificateCredential;
var asTokenCredentials = subscriptionCredentials as HDInsightAccessTokenCredential;
if (asCertificateCredentials.IsNotNull())
{
clientCredential = new JobSubmissionCertificateCredential(asCertificateCredentials, cluster);
}
else if (asTokenCredentials.IsNotNull())
{
clientCredential = new JobSubmissionAccessTokenCredential(asTokenCredentials, cluster);
}
}
return(clientCredential);
}
public static IJobSubmissionClientCredential GetJobSubmissionClientCredentials(this IAzureHDInsightJobCommandCredentialsBase command, WindowsAzureSubscription currentSubscription, string cluster)
{
IJobSubmissionClientCredential clientCredential = null;
if (command.Credential != null)
{
clientCredential = new BasicAuthCredential
{
Server = GatewayUriResolver.GetGatewayUri(cluster),
UserName = command.Credential.UserName,
Password = command.Credential.GetCleartextPassword()
};
}
else if (currentSubscription.IsNotNull())
{
var subscriptionCredentials = GetSubscriptionCredentials(command, currentSubscription);
var asCertificateCredentials = subscriptionCredentials as HDInsightCertificateCredential;
var asTokenCredentials = subscriptionCredentials as HDInsightAccessTokenCredential;
if (asCertificateCredentials.IsNotNull())
{
clientCredential = new JobSubmissionCertificateCredential(asCertificateCredentials, cluster);
}
else if (asTokenCredentials.IsNotNull())
{
clientCredential = new JobSubmissionAccessTokenCredential(asTokenCredentials, cluster);
}
}
return clientCredential;
}
public void ServiceHost_JobSubmissionRecieved(object sender, JobSubmissionMessage e)
{
Trace.WriteLine("JobSubmissionRecieved Recieved User Id : " + e.idUsuario, "Warning");
try
{
// Obtener la cuenta de almacenamiento
// Para actualizar metadatos
CloudStorageAccount storageAccount = CloudStorageAccount.Parse(
CloudConfigurationManager.GetSetting("StorageConnectionString"));
CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient();
CloudBlobContainer container = blobClient.GetContainerReference(VariablesConfiguracion.containerName);
// Obtencion de variables para la conxion con el cluster
string subscriptionID = VariablesConfiguracion.subscriptionID;
string certFriendlyName = VariablesConfiguracion.certFriendlyName;
string clusterName = VariablesConfiguracion.clusterName;
// Definicion de la tarea MapReduce
MapReduceJobCreateParameters mrJobDefinition = new MapReduceJobCreateParameters()
{
JarFile = "wasb:///CienciaCelularMR.jar",
ClassName = "Main",
StatusFolder = "wasb:///scicluster/test/status-" + e.idUsuario + "." + e.subIdUsuario,
};
mrJobDefinition.Arguments.Add("wasb:///" + e.nomEntrada);
mrJobDefinition.Arguments.Add("wasb:///scicluster/test/output-" + e.idUsuario + "." + e.subIdUsuario);
// Obtener el objeto certificate
X509Store store = new X509Store();
store.Open(OpenFlags.ReadOnly);
X509Certificate2 cert = FindCertificate(StoreLocation.CurrentUser, StoreName.My, X509FindType.FindByThumbprint, VariablesConfiguracion.thumbprint);
JobSubmissionCertificateCredential creds = new JobSubmissionCertificateCredential(new Guid(subscriptionID), cert, clusterName);
// Se crea un cliente Hadoop para conectarse con HDInsight
var jobClient = JobSubmissionClientFactory.Connect(creds);
//Actualizacion de metadatos
CloudBlockBlob infoSimulation = container.GetBlockBlobReference(VariablesConfiguracion.infoSimulation + "-" + e.idUsuario);
infoSimulation.UploadText(VariablesConfiguracion.JOB_STARTING);
// Se lanza la ejecucion de los jobs MapReduce
JobCreationResults mrJobResults = jobClient.CreateMapReduceJob(mrJobDefinition);
// Esperar hasta que finalice la ejecucion
WaitForJobCompletion(mrJobResults, jobClient, e.idUsuario, e.subIdUsuario);
}
catch (Exception ex)
{
Trace.TraceError(ex.Message);
throw;
}
}
//Run Sample Map Reduce Job
public static void DoMapReduce()
{
// Define the MapReduce job
MapReduceJobCreateParameters mrJobDefinition = new MapReduceJobCreateParameters()
{
JarFile = "wasb:///example/jars/hadoop-examples.jar",
ClassName = "wordcount"
};
mrJobDefinition.Arguments.Add("wasb:///example/data/gutenberg/davinci.txt");
mrJobDefinition.Arguments.Add("wasb:///example/data/WordCountOutput");
//Get certificate
var store = new X509Store();
store.Open(OpenFlags.ReadOnly);
var cert = store.Certificates.Cast <X509Certificate2>().First(item => item.Thumbprint == Constants.thumbprint);
var creds = new JobSubmissionCertificateCredential(Constants.subscriptionId, cert, Constants.clusterName);
// Create a hadoop client to connect to HDInsight
var jobClient = JobSubmissionClientFactory.Connect(creds);
// Run the MapReduce job
JobCreationResults mrJobResults = jobClient.CreateMapReduceJob(mrJobDefinition);
Console.Write("Executing WordCount MapReduce Job.");
// Wait for the job to complete
WaitForJobCompletion(mrJobResults, jobClient);
// Print the MapReduce job output
Stream stream = new MemoryStream();
CloudStorageAccount storageAccount = CloudStorageAccount.Parse("DefaultEndpointsProtocol=https;AccountName=" + Constants.storageAccount + ";AccountKey=" + Constants.storageAccountKey);
CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient();
CloudBlobContainer blobContainer = blobClient.GetContainerReference(Constants.container);
CloudBlockBlob blockBlob = blobContainer.GetBlockBlobReference("example/data/WordCountOutput/part-r-00000");
blockBlob.DownloadToStream(stream);
stream.Position = 0;
StreamReader reader = new StreamReader(stream);
Console.Write("Done..Word counts are:\n");
Console.WriteLine(reader.ReadToEnd());
}
public AzureYarnClient(AzureSubscriptions subscriptions, AzureDfsClient dfsClient, Uri baseUri,
string ppmHome, string clusterName = null)
{
this.dfsClient = dfsClient;
this.baseUri = baseUri;
this.peloponneseHome = ppmHome;
IEnumerable <AzureCluster> clusters = subscriptions.GetClustersAsync().Result;
AzureCluster cluster;
if (clusterName == null)
{
if (clusters.Count() != 1)
{
throw new ArgumentException("A cluster name must be provided if there is not exactly one configured HDInsight cluster.", "clusterName");
}
cluster = clusters.Single();
}
else
{
IEnumerable <AzureCluster> matching = clusters.Where(c => c.Name == clusterName);
if (matching.Count() == 0)
{
throw new ArgumentException("Cluster " + clusterName + " not attached to a Powershell subscription or specified manuall", "clusterName");
}
cluster = matching.First();
}
ClusterName = cluster.Name;
SubscriptionId = cluster.SubscriptionId;
Guid subscriptionGuid = new Guid(SubscriptionId);
HDInsightCertificateCredential sCred = new HDInsightCertificateCredential(subscriptionGuid, cluster.Certificate);
IHDInsightClient sClient = HDInsightClient.Connect(sCred);
credentials = new JobSubmissionCertificateCredential(sCred, ClusterName);
JobClient = JobSubmissionClientFactory.Connect(credentials);
}
public string HiveOutput(string q)
{
Trace.WriteLine("Entering HiveOutput method");
Trace.TraceInformation("Executing HiveOutput method at " + DateTime.Now.ToLongTimeString());
//Defining MapReduce Job
HiveJobCreateParameters hiveJobDefinition = new HiveJobCreateParameters()
{
JobName = "job",
StatusFolder = "/TableListFolder",
Query = q
};
Guid subscriptionId = new Guid("44fbb137-edbb-4044-9db9-0e1333e137cf"); //your-subscription-id
string clusterName = "tbanihumcluster";
// Get the certificate object from certificate store using the friendly name to identify it
X509Store store = new X509Store(StoreName.My);
store.Open(OpenFlags.ReadOnly);
X509Certificate2 cert = store.Certificates.Cast <X509Certificate2>().First(item => item.FriendlyName == "Azdem187U23713U-1-8-2015-credentials");
var creds = new JobSubmissionCertificateCredential(subscriptionId, cert, clusterName);
// Create a hadoop client to connect to HDInsight
var jobClient = JobSubmissionClientFactory.Connect(creds);
// Run the MapReduce job
JobCreationResults jobResults = jobClient.CreateHiveJob(hiveJobDefinition);
// Wait for the job to complete
WaitForJobCompletion(jobResults, jobClient);
// Hive job output
System.IO.Stream stream = jobClient.GetJobOutput(jobResults.JobId);
System.IO.StreamReader reader = new System.IO.StreamReader(stream);
string value = reader.ReadToEnd();
value = value.Replace('\t', ',');
// Retrieve storage account from connection string.
CloudStorageAccount storageAccount = CloudStorageAccount.Parse(CloudConfigurationManager.GetSetting("StorageConnectionString"));
// Create the blob client.
CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient();
// Retrieve reference to a previously created container.
CloudBlobContainer container = blobClient.GetContainerReference("mycontainer");
container.CreateIfNotExists();
// Retrieve reference to a blob named "myblob".
CloudBlockBlob blockBlob = container.GetBlockBlobReference("myblob");
blockBlob.UploadText(value);
Trace.WriteLine("Leaving HiveOutput method");
return(value);
}
