static void Main(string[] args)
{
var credentials = new BasicAuthCredential
{
UserName = "******",
Password = "******",
Server = new Uri("https://simonellistonball.azurehdinsight.net")
};
var submissionClient = JobSubmissionClientFactory.Connect(credentials);
submissionClient.JobStatusEvent += (o, e) =>
Console.WriteLine(e.JobDetails.StatusCode);
var pigJobParams = new PigJobCreateParameters
{
StatusFolder =
"wasbs://[email protected]/status",
File =
"wasbs://[email protected]/lovecleanstreets.pig"
};
var job = submissionClient.CreatePigJob(pigJobParams);
bool complete = false;
while (!complete)
{
var jobDetails = submissionClient.GetJob(job.JobId);
Console.WriteLine(jobDetails.StatusCode);
Console.WriteLine(jobDetails.ExitCode);
if (jobDetails.StatusCode == JobStatusCode.Completed)
{
complete = true;
}
Thread.Sleep(2000);
}
Console.ReadLine();
}
//Run Hive Job
public static void DoHiveOperations()
{
HiveJobCreateParameters hiveJobDefinition = new HiveJobCreateParameters()
{
JobName = "Show tables job",
StatusFolder = "/TableListFolder",
Query = "show tables;"
};
var store = new X509Store();
store.Open(OpenFlags.ReadOnly);
var cert = store.Certificates.Cast <X509Certificate2>().First(item => item.Thumbprint == Constants.thumbprint);
var creds = new JobSubmissionCertificateCredential(Constants.subscriptionId, cert, Constants.clusterName);
var jobClient = JobSubmissionClientFactory.Connect(creds);
JobCreationResults jobResults = jobClient.CreateHiveJob(hiveJobDefinition);
Console.Write("Executing Hive Job.");
// Wait for the job to complete
WaitForJobCompletion(jobResults, jobClient);
// Print the Hive job output
System.IO.Stream stream = jobClient.GetJobOutput(jobResults.JobId);
System.IO.StreamReader reader = new System.IO.StreamReader(stream);
Console.Write("Done..List of Tables are:\n");
Console.WriteLine(reader.ReadToEnd());
}
public void ServiceHost_JobSubmissionRecieved(object sender, JobSubmissionMessage e)
{
Trace.WriteLine("JobSubmissionRecieved Recieved User Id : " + e.idUsuario, "Warning");
try
{
// Obtener la cuenta de almacenamiento
// Para actualizar metadatos
CloudStorageAccount storageAccount = CloudStorageAccount.Parse(
CloudConfigurationManager.GetSetting("StorageConnectionString"));
CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient();
CloudBlobContainer container = blobClient.GetContainerReference(VariablesConfiguracion.containerName);
// Obtencion de variables para la conxion con el cluster
string subscriptionID = VariablesConfiguracion.subscriptionID;
string certFriendlyName = VariablesConfiguracion.certFriendlyName;
string clusterName = VariablesConfiguracion.clusterName;
// Definicion de la tarea MapReduce
MapReduceJobCreateParameters mrJobDefinition = new MapReduceJobCreateParameters()
{
JarFile = "wasb:///CienciaCelularMR.jar",
ClassName = "Main",
StatusFolder = "wasb:///scicluster/test/status-" + e.idUsuario + "." + e.subIdUsuario,
};
mrJobDefinition.Arguments.Add("wasb:///" + e.nomEntrada);
mrJobDefinition.Arguments.Add("wasb:///scicluster/test/output-" + e.idUsuario + "." + e.subIdUsuario);
// Obtener el objeto certificate
X509Store store = new X509Store();
store.Open(OpenFlags.ReadOnly);
X509Certificate2 cert = FindCertificate(StoreLocation.CurrentUser, StoreName.My, X509FindType.FindByThumbprint, VariablesConfiguracion.thumbprint);
JobSubmissionCertificateCredential creds = new JobSubmissionCertificateCredential(new Guid(subscriptionID), cert, clusterName);
// Se crea un cliente Hadoop para conectarse con HDInsight
var jobClient = JobSubmissionClientFactory.Connect(creds);
//Actualizacion de metadatos
CloudBlockBlob infoSimulation = container.GetBlockBlobReference(VariablesConfiguracion.infoSimulation + "-" + e.idUsuario);
infoSimulation.UploadText(VariablesConfiguracion.JOB_STARTING);
// Se lanza la ejecucion de los jobs MapReduce
JobCreationResults mrJobResults = jobClient.CreateMapReduceJob(mrJobDefinition);
// Esperar hasta que finalice la ejecucion
WaitForJobCompletion(mrJobResults, jobClient, e.idUsuario, e.subIdUsuario);
}
catch (Exception ex)
{
Trace.TraceError(ex.Message);
throw;
}
}
//Run Sample Map Reduce Job
public static void DoMapReduce()
{
// Define the MapReduce job
MapReduceJobCreateParameters mrJobDefinition = new MapReduceJobCreateParameters()
{
JarFile = "wasb:///example/jars/hadoop-examples.jar",
ClassName = "wordcount"
};
mrJobDefinition.Arguments.Add("wasb:///example/data/gutenberg/davinci.txt");
mrJobDefinition.Arguments.Add("wasb:///example/data/WordCountOutput");
//Get certificate
var store = new X509Store();
store.Open(OpenFlags.ReadOnly);
var cert = store.Certificates.Cast <X509Certificate2>().First(item => item.Thumbprint == Constants.thumbprint);
var creds = new JobSubmissionCertificateCredential(Constants.subscriptionId, cert, Constants.clusterName);
// Create a hadoop client to connect to HDInsight
var jobClient = JobSubmissionClientFactory.Connect(creds);
// Run the MapReduce job
JobCreationResults mrJobResults = jobClient.CreateMapReduceJob(mrJobDefinition);
Console.Write("Executing WordCount MapReduce Job.");
// Wait for the job to complete
WaitForJobCompletion(mrJobResults, jobClient);
// Print the MapReduce job output
Stream stream = new MemoryStream();
CloudStorageAccount storageAccount = CloudStorageAccount.Parse("DefaultEndpointsProtocol=https;AccountName=" + Constants.storageAccount + ";AccountKey=" + Constants.storageAccountKey);
CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient();
CloudBlobContainer blobContainer = blobClient.GetContainerReference(Constants.container);
CloudBlockBlob blockBlob = blobContainer.GetBlockBlobReference("example/data/WordCountOutput/part-r-00000");
blockBlob.DownloadToStream(stream);
stream.Position = 0;
StreamReader reader = new StreamReader(stream);
Console.Write("Done..Word counts are:\n");
Console.WriteLine(reader.ReadToEnd());
}
/// <summary>
/// Connects to HDInsight cluster.
/// </summary>
/// <param name="certificate">The certificate.</param>
/// <param name="subscription">The subscription.</param>
/// <param name="clusterName">Name of the cluster.</param>
/// <param name="storageAccountName">Name of the storage account.</param>
/// <param name="storageAccountKey">The storage account key.</param>
public void Connect(string certificate, string subscription, string clusterName, string storageAccountName, string storageAccountKey)
{
// Obtain the certificate
var store = new X509Store();
store.Open(OpenFlags.ReadOnly);
var cert = store.Certificates.Cast <X509Certificate2>().FirstOrDefault(item => string.Compare(item.Thumbprint, certificate, CultureInfo.InvariantCulture, CompareOptions.IgnoreCase) == 0);
if (cert == null)
{
AvroHdiSample.ReportError("Error: Counld not find the certificate on this machine!");
}
// Connect to the cluster using the certificate and the subscription
try
{
this.client = HDInsightClient.Connect(new HDInsightCertificateCredential(new Guid(subscription), cert));
}
catch (Exception e)
{
AvroHdiSample.ReportError("Error while connecting to HDInsight service\n" + e);
}
this.cluster = this.client.GetCluster(clusterName);
if (this.cluster == null)
{
AvroHdiSample.ReportError("Error while connecting to cluster: " + clusterName);
}
// Create a job client
this.job = JobSubmissionClientFactory.Connect(
new JobSubmissionCertificateCredential(new Guid(subscription), cert, clusterName));
// Create an Azure storage client
// We will use this client to upload files to Azure storage account
// which is used by HDInsight cluster.
var storageAccount = CloudStorageAccount.Parse(
"DefaultEndpointsProtocol=https;AccountName=" + storageAccountName + ";AccountKey=" + storageAccountKey);
var blobClient = storageAccount.CreateCloudBlobClient();
this.container = blobClient.GetContainerReference(this.cluster.DefaultStorageAccount.Container);
}
public AzureYarnClient(AzureSubscriptions subscriptions, AzureDfsClient dfsClient, Uri baseUri,
string ppmHome, string clusterName = null)
{
this.dfsClient = dfsClient;
this.baseUri = baseUri;
this.peloponneseHome = ppmHome;
IEnumerable <AzureCluster> clusters = subscriptions.GetClustersAsync().Result;
AzureCluster cluster;
if (clusterName == null)
{
if (clusters.Count() != 1)
{
throw new ArgumentException("A cluster name must be provided if there is not exactly one configured HDInsight cluster.", "clusterName");
}
cluster = clusters.Single();
}
else
{
IEnumerable <AzureCluster> matching = clusters.Where(c => c.Name == clusterName);
if (matching.Count() == 0)
{
throw new ArgumentException("Cluster " + clusterName + " not attached to a Powershell subscription or specified manuall", "clusterName");
}
cluster = matching.First();
}
ClusterName = cluster.Name;
SubscriptionId = cluster.SubscriptionId;
Guid subscriptionGuid = new Guid(SubscriptionId);
HDInsightCertificateCredential sCred = new HDInsightCertificateCredential(subscriptionGuid, cluster.Certificate);
IHDInsightClient sClient = HDInsightClient.Connect(sCred);
credentials = new JobSubmissionCertificateCredential(sCred, ClusterName);
JobClient = JobSubmissionClientFactory.Connect(credentials);
}
private static void SubmitJobs()
{
// Get HDInsight cluster configuration settings
string clusterName = ConfigurationManager.AppSettings["ClusterName"];
string userName = ConfigurationManager.AppSettings["UserName"];
string password = ConfigurationManager.AppSettings["Password"];
// Create basic authentication credential for cluster
BasicAuthCredential bcred = new BasicAuthCredential();
bcred.Server = new Uri("https://" + clusterName + ".azurehdinsight.net");
bcred.UserName = userName;
bcred.Password = password;
// Create and submit Pig job
PigJobCreateParameters pigJob = new PigJobCreateParameters()
{
StatusFolder = "/data/racecar/scripts/processdatastatus",
File = "/data/racecar/scripts/processdata.pig"
};
var pigJobClient = JobSubmissionClientFactory.Connect(bcred);
JobCreationResults pigJobResults = pigJobClient.CreatePigJob(pigJob);
WaitForJobCompletion(pigJobResults, pigJobClient);
// Create and submit Hive job
HiveJobCreateParameters hiveJob = new HiveJobCreateParameters()
{
JobName = "Create Hive tables",
StatusFolder = "/data/racecar/scripts/createtablestatus",
File = "/data/racecar/scripts/createtables.hql"
};
var hiveJobClient = JobSubmissionClientFactory.Connect(bcred);
JobCreationResults hiveJobResults = hiveJobClient.CreateHiveJob(hiveJob);
WaitForJobCompletion(hiveJobResults, hiveJobClient);
}
public IJobSubmissionClient Create(IJobSubmissionClientCredential credentials)
{
return(JobSubmissionClientFactory.Connect(credentials));
}
public void ICanNotSubmitAJobWithTheIncorectCredintials()
{
IHDInsightCertificateCredential hdInsightCredentials = IntegrationTestBase.GetValidCredentials();
var client = ServiceLocator.Instance.Locate <IHDInsightClientFactory>().Create(new HDInsightCertificateCredential(hdInsightCredentials.SubscriptionId, hdInsightCredentials.Certificate));
var manager = ServiceLocator.Instance.Locate <IHDInsightManagementPocoClientFactory>();
var pocoClient = manager.Create(hdInsightCredentials, GetAbstractionContext(), false);
var clusterDetails = GetRandomCluster();
client.CreateCluster(clusterDetails);
try
{
ClusterDetails cluster = pocoClient.ListContainer(clusterDetails.Name).WaitForResult();
BasicAuthCredential hadoopCredentials = new BasicAuthCredential()
{
Server = GatewayUriResolver.GetGatewayUri(cluster.ConnectionUrl),
UserName = clusterDetails.UserName,
Password = clusterDetails.Password
};
var hadoopClient = JobSubmissionClientFactory.Connect(hadoopCredentials);
var mapReduceJob = new MapReduceJobCreateParameters()
{
ClassName = "pi",
JobName = "pi estimation jobDetails",
JarFile = "/example/hadoop-examples.jar",
StatusFolder = "/piresults"
};
mapReduceJob.Arguments.Add("16");
mapReduceJob.Arguments.Add("10000");
var jobCreationDetails = hadoopClient.CreateMapReduceJob(mapReduceJob);
var id = pocoClient.DisableHttp(clusterDetails.Name, clusterDetails.Location).WaitForResult();
while (!pocoClient.IsComplete(cluster.Name, cluster.Location, id).WaitForResult())
{
Thread.Sleep(500);
}
// now add a user
string userName = "******";
string password = GetRandomValidPassword();
id = pocoClient.EnableHttp(clusterDetails.Name, clusterDetails.Location, userName, password).WaitForResult();
while (!pocoClient.IsComplete(cluster.Name, cluster.Location, id).WaitForResult())
{
Thread.Sleep(500);
}
jobCreationDetails = hadoopClient.CreateMapReduceJob(mapReduceJob);
Assert.Fail("This test expected an exception but did not receive one.");
}
catch (UnauthorizedAccessException ex)
{
Help.DoNothing(ex);
}
finally
{
// delete the cluster
client.DeleteCluster(clusterDetails.Name);
}
}
public string HiveOutput(string q)
{
Trace.WriteLine("Entering HiveOutput method");
Trace.TraceInformation("Executing HiveOutput method at " + DateTime.Now.ToLongTimeString());
//Defining MapReduce Job
HiveJobCreateParameters hiveJobDefinition = new HiveJobCreateParameters()
{
JobName = "job",
StatusFolder = "/TableListFolder",
Query = q
};
Guid subscriptionId = new Guid("44fbb137-edbb-4044-9db9-0e1333e137cf"); //your-subscription-id
string clusterName = "tbanihumcluster";
// Get the certificate object from certificate store using the friendly name to identify it
X509Store store = new X509Store(StoreName.My);
store.Open(OpenFlags.ReadOnly);
X509Certificate2 cert = store.Certificates.Cast <X509Certificate2>().First(item => item.FriendlyName == "Azdem187U23713U-1-8-2015-credentials");
var creds = new JobSubmissionCertificateCredential(subscriptionId, cert, clusterName);
// Create a hadoop client to connect to HDInsight
var jobClient = JobSubmissionClientFactory.Connect(creds);
// Run the MapReduce job
JobCreationResults jobResults = jobClient.CreateHiveJob(hiveJobDefinition);
// Wait for the job to complete
WaitForJobCompletion(jobResults, jobClient);
// Hive job output
System.IO.Stream stream = jobClient.GetJobOutput(jobResults.JobId);
System.IO.StreamReader reader = new System.IO.StreamReader(stream);
string value = reader.ReadToEnd();
value = value.Replace('\t', ',');
// Retrieve storage account from connection string.
CloudStorageAccount storageAccount = CloudStorageAccount.Parse(CloudConfigurationManager.GetSetting("StorageConnectionString"));
// Create the blob client.
CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient();
// Retrieve reference to a previously created container.
CloudBlobContainer container = blobClient.GetContainerReference("mycontainer");
container.CreateIfNotExists();
// Retrieve reference to a blob named "myblob".
CloudBlockBlob blockBlob = container.GetBlockBlobReference("myblob");
blockBlob.UploadText(value);
Trace.WriteLine("Leaving HiveOutput method");
return(value);
}
/// <inheritdoc />
private async Task DownloadApplicationLogsAsync(string applicationId, string applicationUser, string containerId, string nodeId, string targetDirectory)
{
applicationId.ArgumentNotNullOrEmpty("applicationId");
applicationUser.ArgumentNotNullOrEmpty("applicationUser");
targetDirectory.ArgumentNotNullOrEmpty("targetDirectory");
if (!string.IsNullOrEmpty(containerId) && string.IsNullOrEmpty(nodeId))
{
throw new ArgumentException("NodeId was null or empty. If container id is specified, node id should also be specified");
}
if (!Directory.Exists(targetDirectory))
{
throw new ArgumentException(string.Format(CultureInfo.InvariantCulture, "The specified directory {0} does not exist.", targetDirectory));
}
var jobSubmissionClient = JobSubmissionClientFactory.Connect(this.HttpCredentials, this.customUserAgent);
var storageClient = ServiceLocator.Instance.Locate <IWabStorageAbstractionFactory>().Create(this.DefaultStorageCredentials);
// Check existence of application logs in the default storage account
Uri appLogContainer = new Uri(string.Format(CultureInfo.InvariantCulture, "{0}{1}@{2}/app-logs/{3}/logs/{4}", Constants.WabsProtocolSchemeName, this.DefaultStorageCredentials.ContainerName, this.DefaultStorageCredentials.Name, applicationUser, applicationId));
var logFiles = await storageClient.List(appLogContainer, false);
if (!logFiles.Any())
{
throw new InvalidOperationException(string.Format("No logs found for application id {0}, user {1}, on cluster {2} at location {3}", applicationId, applicationUser, this.Cluster.Name, appLogContainer.AbsoluteUri));
}
// Application logs exist!
// Convert them to plain text by running YARN CLI
string jobName = string.Format("yarnlogs-{0}", Guid.NewGuid());
string statusFolderName = string.Format("/{0}", jobName);
string optionalContainerArguments = !string.IsNullOrEmpty(containerId) ? string.Format(" -containerId {0} -nodeAddress {1}", containerId, nodeId) : string.Empty;
string command = "";
if (this.Cluster.OSType == OSType.Windows)
{
command = string.Format("!cmd.exe /c yarn logs -applicationId {0} -appOwner {1}{2};", applicationId, applicationUser, optionalContainerArguments);
}
else if (this.Cluster.OSType == OSType.Linux)
{
command = string.Format("!yarn logs -applicationId {0} -appOwner {1}{2};", applicationId, applicationUser, optionalContainerArguments);
}
else
{
throw new NotSupportedException(String.Format("This functionality is not supported on clusters with OS Type: {0}", this.Cluster.OSType));
}
string queryFileName = string.Format("/{0}.hql", jobName);
Uri queryFileUri = new Uri(string.Format(CultureInfo.InvariantCulture, "{0}{1}@{2}{3}", Constants.WabsProtocolSchemeName, this.DefaultStorageCredentials.ContainerName, this.DefaultStorageCredentials.Name, queryFileName));
Uri statusFolderUri = new Uri(string.Format(CultureInfo.InvariantCulture, "{0}{1}@{2}{3}", Constants.WabsProtocolSchemeName, this.DefaultStorageCredentials.ContainerName, this.DefaultStorageCredentials.Name, statusFolderName));
try
{
var bytes = Encoding.UTF8.GetBytes(command);
using (var memoryStream = new MemoryStream(bytes))
{
await storageClient.Write(queryFileUri, memoryStream);
}
HiveJobCreateParameters hiveJobDefinition = new HiveJobCreateParameters()
{
JobName = jobName,
StatusFolder = statusFolderName,
File = queryFileName
};
JobCreationResults jobResults = jobSubmissionClient.CreateHiveJob(hiveJobDefinition);
WaitForJobCompletion(jobSubmissionClient, jobResults);
Uri logContentsFileUri = new Uri(string.Format("{0}/stdout", statusFolderUri.AbsoluteUri));
if (await storageClient.Exists(logContentsFileUri))
{
// create local file in the targetdirectory.
var localFilePath = Path.Combine(targetDirectory, string.Format("{0}_{1}.txt", this.Cluster.Name, string.IsNullOrEmpty(containerId) ? applicationId : containerId));
await storageClient.DownloadToFile(logContentsFileUri, localFilePath);
}
else
{
throw new InvalidOperationException(string.Format(
CultureInfo.InvariantCulture,
"Could not retrive logs for application id {0}, user {1} on cluster {2} at location {3}.",
applicationId,
applicationUser,
this.Cluster.Name,
appLogContainer.AbsoluteUri));
}
}
finally
{
// Cleanup what we created
if (storageClient.Exists(queryFileUri).WaitForResult())
{
storageClient.Delete(queryFileUri);
}
if (storageClient.Exists(statusFolderUri).WaitForResult())
{
storageClient.Delete(statusFolderUri);
}
}
}