static void Main(string[] args) { var credentials = new BasicAuthCredential { UserName = "******", Password = "******", Server = new Uri("https://simonellistonball.azurehdinsight.net") }; var submissionClient = JobSubmissionClientFactory.Connect(credentials); submissionClient.JobStatusEvent += (o, e) => Console.WriteLine(e.JobDetails.StatusCode); var pigJobParams = new PigJobCreateParameters { StatusFolder = "wasbs://[email protected]/status", File = "wasbs://[email protected]/lovecleanstreets.pig" }; var job = submissionClient.CreatePigJob(pigJobParams); bool complete = false; while (!complete) { var jobDetails = submissionClient.GetJob(job.JobId); Console.WriteLine(jobDetails.StatusCode); Console.WriteLine(jobDetails.ExitCode); if (jobDetails.StatusCode == JobStatusCode.Completed) { complete = true; } Thread.Sleep(2000); } Console.ReadLine(); }
//Run Hive Job public static void DoHiveOperations() { HiveJobCreateParameters hiveJobDefinition = new HiveJobCreateParameters() { JobName = "Show tables job", StatusFolder = "/TableListFolder", Query = "show tables;" }; var store = new X509Store(); store.Open(OpenFlags.ReadOnly); var cert = store.Certificates.Cast <X509Certificate2>().First(item => item.Thumbprint == Constants.thumbprint); var creds = new JobSubmissionCertificateCredential(Constants.subscriptionId, cert, Constants.clusterName); var jobClient = JobSubmissionClientFactory.Connect(creds); JobCreationResults jobResults = jobClient.CreateHiveJob(hiveJobDefinition); Console.Write("Executing Hive Job."); // Wait for the job to complete WaitForJobCompletion(jobResults, jobClient); // Print the Hive job output System.IO.Stream stream = jobClient.GetJobOutput(jobResults.JobId); System.IO.StreamReader reader = new System.IO.StreamReader(stream); Console.Write("Done..List of Tables are:\n"); Console.WriteLine(reader.ReadToEnd()); }
public void ServiceHost_JobSubmissionRecieved(object sender, JobSubmissionMessage e) { Trace.WriteLine("JobSubmissionRecieved Recieved User Id : " + e.idUsuario, "Warning"); try { // Obtener la cuenta de almacenamiento // Para actualizar metadatos CloudStorageAccount storageAccount = CloudStorageAccount.Parse( CloudConfigurationManager.GetSetting("StorageConnectionString")); CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient(); CloudBlobContainer container = blobClient.GetContainerReference(VariablesConfiguracion.containerName); // Obtencion de variables para la conxion con el cluster string subscriptionID = VariablesConfiguracion.subscriptionID; string certFriendlyName = VariablesConfiguracion.certFriendlyName; string clusterName = VariablesConfiguracion.clusterName; // Definicion de la tarea MapReduce MapReduceJobCreateParameters mrJobDefinition = new MapReduceJobCreateParameters() { JarFile = "wasb:///CienciaCelularMR.jar", ClassName = "Main", StatusFolder = "wasb:///scicluster/test/status-" + e.idUsuario + "." + e.subIdUsuario, }; mrJobDefinition.Arguments.Add("wasb:///" + e.nomEntrada); mrJobDefinition.Arguments.Add("wasb:///scicluster/test/output-" + e.idUsuario + "." + e.subIdUsuario); // Obtener el objeto certificate X509Store store = new X509Store(); store.Open(OpenFlags.ReadOnly); X509Certificate2 cert = FindCertificate(StoreLocation.CurrentUser, StoreName.My, X509FindType.FindByThumbprint, VariablesConfiguracion.thumbprint); JobSubmissionCertificateCredential creds = new JobSubmissionCertificateCredential(new Guid(subscriptionID), cert, clusterName); // Se crea un cliente Hadoop para conectarse con HDInsight var jobClient = JobSubmissionClientFactory.Connect(creds); //Actualizacion de metadatos CloudBlockBlob infoSimulation = container.GetBlockBlobReference(VariablesConfiguracion.infoSimulation + "-" + e.idUsuario); infoSimulation.UploadText(VariablesConfiguracion.JOB_STARTING); // Se lanza la ejecucion de los jobs MapReduce JobCreationResults mrJobResults = jobClient.CreateMapReduceJob(mrJobDefinition); // Esperar hasta que finalice la ejecucion WaitForJobCompletion(mrJobResults, jobClient, e.idUsuario, e.subIdUsuario); } catch (Exception ex) { Trace.TraceError(ex.Message); throw; } }
//Run Sample Map Reduce Job public static void DoMapReduce() { // Define the MapReduce job MapReduceJobCreateParameters mrJobDefinition = new MapReduceJobCreateParameters() { JarFile = "wasb:///example/jars/hadoop-examples.jar", ClassName = "wordcount" }; mrJobDefinition.Arguments.Add("wasb:///example/data/gutenberg/davinci.txt"); mrJobDefinition.Arguments.Add("wasb:///example/data/WordCountOutput"); //Get certificate var store = new X509Store(); store.Open(OpenFlags.ReadOnly); var cert = store.Certificates.Cast <X509Certificate2>().First(item => item.Thumbprint == Constants.thumbprint); var creds = new JobSubmissionCertificateCredential(Constants.subscriptionId, cert, Constants.clusterName); // Create a hadoop client to connect to HDInsight var jobClient = JobSubmissionClientFactory.Connect(creds); // Run the MapReduce job JobCreationResults mrJobResults = jobClient.CreateMapReduceJob(mrJobDefinition); Console.Write("Executing WordCount MapReduce Job."); // Wait for the job to complete WaitForJobCompletion(mrJobResults, jobClient); // Print the MapReduce job output Stream stream = new MemoryStream(); CloudStorageAccount storageAccount = CloudStorageAccount.Parse("DefaultEndpointsProtocol=https;AccountName=" + Constants.storageAccount + ";AccountKey=" + Constants.storageAccountKey); CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient(); CloudBlobContainer blobContainer = blobClient.GetContainerReference(Constants.container); CloudBlockBlob blockBlob = blobContainer.GetBlockBlobReference("example/data/WordCountOutput/part-r-00000"); blockBlob.DownloadToStream(stream); stream.Position = 0; StreamReader reader = new StreamReader(stream); Console.Write("Done..Word counts are:\n"); Console.WriteLine(reader.ReadToEnd()); }
/// <summary> /// Connects to HDInsight cluster. /// </summary> /// <param name="certificate">The certificate.</param> /// <param name="subscription">The subscription.</param> /// <param name="clusterName">Name of the cluster.</param> /// <param name="storageAccountName">Name of the storage account.</param> /// <param name="storageAccountKey">The storage account key.</param> public void Connect(string certificate, string subscription, string clusterName, string storageAccountName, string storageAccountKey) { // Obtain the certificate var store = new X509Store(); store.Open(OpenFlags.ReadOnly); var cert = store.Certificates.Cast <X509Certificate2>().FirstOrDefault(item => string.Compare(item.Thumbprint, certificate, CultureInfo.InvariantCulture, CompareOptions.IgnoreCase) == 0); if (cert == null) { AvroHdiSample.ReportError("Error: Counld not find the certificate on this machine!"); } // Connect to the cluster using the certificate and the subscription try { this.client = HDInsightClient.Connect(new HDInsightCertificateCredential(new Guid(subscription), cert)); } catch (Exception e) { AvroHdiSample.ReportError("Error while connecting to HDInsight service\n" + e); } this.cluster = this.client.GetCluster(clusterName); if (this.cluster == null) { AvroHdiSample.ReportError("Error while connecting to cluster: " + clusterName); } // Create a job client this.job = JobSubmissionClientFactory.Connect( new JobSubmissionCertificateCredential(new Guid(subscription), cert, clusterName)); // Create an Azure storage client // We will use this client to upload files to Azure storage account // which is used by HDInsight cluster. var storageAccount = CloudStorageAccount.Parse( "DefaultEndpointsProtocol=https;AccountName=" + storageAccountName + ";AccountKey=" + storageAccountKey); var blobClient = storageAccount.CreateCloudBlobClient(); this.container = blobClient.GetContainerReference(this.cluster.DefaultStorageAccount.Container); }
public AzureYarnClient(AzureSubscriptions subscriptions, AzureDfsClient dfsClient, Uri baseUri, string ppmHome, string clusterName = null) { this.dfsClient = dfsClient; this.baseUri = baseUri; this.peloponneseHome = ppmHome; IEnumerable <AzureCluster> clusters = subscriptions.GetClustersAsync().Result; AzureCluster cluster; if (clusterName == null) { if (clusters.Count() != 1) { throw new ArgumentException("A cluster name must be provided if there is not exactly one configured HDInsight cluster.", "clusterName"); } cluster = clusters.Single(); } else { IEnumerable <AzureCluster> matching = clusters.Where(c => c.Name == clusterName); if (matching.Count() == 0) { throw new ArgumentException("Cluster " + clusterName + " not attached to a Powershell subscription or specified manuall", "clusterName"); } cluster = matching.First(); } ClusterName = cluster.Name; SubscriptionId = cluster.SubscriptionId; Guid subscriptionGuid = new Guid(SubscriptionId); HDInsightCertificateCredential sCred = new HDInsightCertificateCredential(subscriptionGuid, cluster.Certificate); IHDInsightClient sClient = HDInsightClient.Connect(sCred); credentials = new JobSubmissionCertificateCredential(sCred, ClusterName); JobClient = JobSubmissionClientFactory.Connect(credentials); }
private static void SubmitJobs() { // Get HDInsight cluster configuration settings string clusterName = ConfigurationManager.AppSettings["ClusterName"]; string userName = ConfigurationManager.AppSettings["UserName"]; string password = ConfigurationManager.AppSettings["Password"]; // Create basic authentication credential for cluster BasicAuthCredential bcred = new BasicAuthCredential(); bcred.Server = new Uri("https://" + clusterName + ".azurehdinsight.net"); bcred.UserName = userName; bcred.Password = password; // Create and submit Pig job PigJobCreateParameters pigJob = new PigJobCreateParameters() { StatusFolder = "/data/racecar/scripts/processdatastatus", File = "/data/racecar/scripts/processdata.pig" }; var pigJobClient = JobSubmissionClientFactory.Connect(bcred); JobCreationResults pigJobResults = pigJobClient.CreatePigJob(pigJob); WaitForJobCompletion(pigJobResults, pigJobClient); // Create and submit Hive job HiveJobCreateParameters hiveJob = new HiveJobCreateParameters() { JobName = "Create Hive tables", StatusFolder = "/data/racecar/scripts/createtablestatus", File = "/data/racecar/scripts/createtables.hql" }; var hiveJobClient = JobSubmissionClientFactory.Connect(bcred); JobCreationResults hiveJobResults = hiveJobClient.CreateHiveJob(hiveJob); WaitForJobCompletion(hiveJobResults, hiveJobClient); }
public IJobSubmissionClient Create(IJobSubmissionClientCredential credentials) { return(JobSubmissionClientFactory.Connect(credentials)); }
public void ICanNotSubmitAJobWithTheIncorectCredintials() { IHDInsightCertificateCredential hdInsightCredentials = IntegrationTestBase.GetValidCredentials(); var client = ServiceLocator.Instance.Locate <IHDInsightClientFactory>().Create(new HDInsightCertificateCredential(hdInsightCredentials.SubscriptionId, hdInsightCredentials.Certificate)); var manager = ServiceLocator.Instance.Locate <IHDInsightManagementPocoClientFactory>(); var pocoClient = manager.Create(hdInsightCredentials, GetAbstractionContext(), false); var clusterDetails = GetRandomCluster(); client.CreateCluster(clusterDetails); try { ClusterDetails cluster = pocoClient.ListContainer(clusterDetails.Name).WaitForResult(); BasicAuthCredential hadoopCredentials = new BasicAuthCredential() { Server = GatewayUriResolver.GetGatewayUri(cluster.ConnectionUrl), UserName = clusterDetails.UserName, Password = clusterDetails.Password }; var hadoopClient = JobSubmissionClientFactory.Connect(hadoopCredentials); var mapReduceJob = new MapReduceJobCreateParameters() { ClassName = "pi", JobName = "pi estimation jobDetails", JarFile = "/example/hadoop-examples.jar", StatusFolder = "/piresults" }; mapReduceJob.Arguments.Add("16"); mapReduceJob.Arguments.Add("10000"); var jobCreationDetails = hadoopClient.CreateMapReduceJob(mapReduceJob); var id = pocoClient.DisableHttp(clusterDetails.Name, clusterDetails.Location).WaitForResult(); while (!pocoClient.IsComplete(cluster.Name, cluster.Location, id).WaitForResult()) { Thread.Sleep(500); } // now add a user string userName = "******"; string password = GetRandomValidPassword(); id = pocoClient.EnableHttp(clusterDetails.Name, clusterDetails.Location, userName, password).WaitForResult(); while (!pocoClient.IsComplete(cluster.Name, cluster.Location, id).WaitForResult()) { Thread.Sleep(500); } jobCreationDetails = hadoopClient.CreateMapReduceJob(mapReduceJob); Assert.Fail("This test expected an exception but did not receive one."); } catch (UnauthorizedAccessException ex) { Help.DoNothing(ex); } finally { // delete the cluster client.DeleteCluster(clusterDetails.Name); } }
public string HiveOutput(string q) { Trace.WriteLine("Entering HiveOutput method"); Trace.TraceInformation("Executing HiveOutput method at " + DateTime.Now.ToLongTimeString()); //Defining MapReduce Job HiveJobCreateParameters hiveJobDefinition = new HiveJobCreateParameters() { JobName = "job", StatusFolder = "/TableListFolder", Query = q }; Guid subscriptionId = new Guid("44fbb137-edbb-4044-9db9-0e1333e137cf"); //your-subscription-id string clusterName = "tbanihumcluster"; // Get the certificate object from certificate store using the friendly name to identify it X509Store store = new X509Store(StoreName.My); store.Open(OpenFlags.ReadOnly); X509Certificate2 cert = store.Certificates.Cast <X509Certificate2>().First(item => item.FriendlyName == "Azdem187U23713U-1-8-2015-credentials"); var creds = new JobSubmissionCertificateCredential(subscriptionId, cert, clusterName); // Create a hadoop client to connect to HDInsight var jobClient = JobSubmissionClientFactory.Connect(creds); // Run the MapReduce job JobCreationResults jobResults = jobClient.CreateHiveJob(hiveJobDefinition); // Wait for the job to complete WaitForJobCompletion(jobResults, jobClient); // Hive job output System.IO.Stream stream = jobClient.GetJobOutput(jobResults.JobId); System.IO.StreamReader reader = new System.IO.StreamReader(stream); string value = reader.ReadToEnd(); value = value.Replace('\t', ','); // Retrieve storage account from connection string. CloudStorageAccount storageAccount = CloudStorageAccount.Parse(CloudConfigurationManager.GetSetting("StorageConnectionString")); // Create the blob client. CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient(); // Retrieve reference to a previously created container. CloudBlobContainer container = blobClient.GetContainerReference("mycontainer"); container.CreateIfNotExists(); // Retrieve reference to a blob named "myblob". CloudBlockBlob blockBlob = container.GetBlockBlobReference("myblob"); blockBlob.UploadText(value); Trace.WriteLine("Leaving HiveOutput method"); return(value); }
/// <inheritdoc /> private async Task DownloadApplicationLogsAsync(string applicationId, string applicationUser, string containerId, string nodeId, string targetDirectory) { applicationId.ArgumentNotNullOrEmpty("applicationId"); applicationUser.ArgumentNotNullOrEmpty("applicationUser"); targetDirectory.ArgumentNotNullOrEmpty("targetDirectory"); if (!string.IsNullOrEmpty(containerId) && string.IsNullOrEmpty(nodeId)) { throw new ArgumentException("NodeId was null or empty. If container id is specified, node id should also be specified"); } if (!Directory.Exists(targetDirectory)) { throw new ArgumentException(string.Format(CultureInfo.InvariantCulture, "The specified directory {0} does not exist.", targetDirectory)); } var jobSubmissionClient = JobSubmissionClientFactory.Connect(this.HttpCredentials, this.customUserAgent); var storageClient = ServiceLocator.Instance.Locate <IWabStorageAbstractionFactory>().Create(this.DefaultStorageCredentials); // Check existence of application logs in the default storage account Uri appLogContainer = new Uri(string.Format(CultureInfo.InvariantCulture, "{0}{1}@{2}/app-logs/{3}/logs/{4}", Constants.WabsProtocolSchemeName, this.DefaultStorageCredentials.ContainerName, this.DefaultStorageCredentials.Name, applicationUser, applicationId)); var logFiles = await storageClient.List(appLogContainer, false); if (!logFiles.Any()) { throw new InvalidOperationException(string.Format("No logs found for application id {0}, user {1}, on cluster {2} at location {3}", applicationId, applicationUser, this.Cluster.Name, appLogContainer.AbsoluteUri)); } // Application logs exist! // Convert them to plain text by running YARN CLI string jobName = string.Format("yarnlogs-{0}", Guid.NewGuid()); string statusFolderName = string.Format("/{0}", jobName); string optionalContainerArguments = !string.IsNullOrEmpty(containerId) ? string.Format(" -containerId {0} -nodeAddress {1}", containerId, nodeId) : string.Empty; string command = ""; if (this.Cluster.OSType == OSType.Windows) { command = string.Format("!cmd.exe /c yarn logs -applicationId {0} -appOwner {1}{2};", applicationId, applicationUser, optionalContainerArguments); } else if (this.Cluster.OSType == OSType.Linux) { command = string.Format("!yarn logs -applicationId {0} -appOwner {1}{2};", applicationId, applicationUser, optionalContainerArguments); } else { throw new NotSupportedException(String.Format("This functionality is not supported on clusters with OS Type: {0}", this.Cluster.OSType)); } string queryFileName = string.Format("/{0}.hql", jobName); Uri queryFileUri = new Uri(string.Format(CultureInfo.InvariantCulture, "{0}{1}@{2}{3}", Constants.WabsProtocolSchemeName, this.DefaultStorageCredentials.ContainerName, this.DefaultStorageCredentials.Name, queryFileName)); Uri statusFolderUri = new Uri(string.Format(CultureInfo.InvariantCulture, "{0}{1}@{2}{3}", Constants.WabsProtocolSchemeName, this.DefaultStorageCredentials.ContainerName, this.DefaultStorageCredentials.Name, statusFolderName)); try { var bytes = Encoding.UTF8.GetBytes(command); using (var memoryStream = new MemoryStream(bytes)) { await storageClient.Write(queryFileUri, memoryStream); } HiveJobCreateParameters hiveJobDefinition = new HiveJobCreateParameters() { JobName = jobName, StatusFolder = statusFolderName, File = queryFileName }; JobCreationResults jobResults = jobSubmissionClient.CreateHiveJob(hiveJobDefinition); WaitForJobCompletion(jobSubmissionClient, jobResults); Uri logContentsFileUri = new Uri(string.Format("{0}/stdout", statusFolderUri.AbsoluteUri)); if (await storageClient.Exists(logContentsFileUri)) { // create local file in the targetdirectory. var localFilePath = Path.Combine(targetDirectory, string.Format("{0}_{1}.txt", this.Cluster.Name, string.IsNullOrEmpty(containerId) ? applicationId : containerId)); await storageClient.DownloadToFile(logContentsFileUri, localFilePath); } else { throw new InvalidOperationException(string.Format( CultureInfo.InvariantCulture, "Could not retrive logs for application id {0}, user {1} on cluster {2} at location {3}.", applicationId, applicationUser, this.Cluster.Name, appLogContainer.AbsoluteUri)); } } finally { // Cleanup what we created if (storageClient.Exists(queryFileUri).WaitForResult()) { storageClient.Delete(queryFileUri); } if (storageClient.Exists(statusFolderUri).WaitForResult()) { storageClient.Delete(statusFolderUri); } } }