private JobCreationResults CreateJobSuccessResult(JobDetails jobDetailsHistoryEntry, string jobName) { if (this.cluster.IsNull()) { throw new InvalidOperationException("The cluster could not be found."); } //if (this.credentials.UserName != this.cluster.Cluster.HttpUserName && this.credentials.Password != this.cluster.Cluster.HttpPassword) //{ // throw new UnauthorizedAccessException("The supplied credential do not have access to the server."); //} lock (this.cluster.JobQueue) { this.LogMessage("Starting jobDetails '{0}'.", jobName); var jobCreationResults = new JobCreationResults { JobId = "job_" + Guid.NewGuid().ToString(), HttpStatusCode = HttpStatusCode.OK }; jobDetailsHistoryEntry.Name = jobName; jobDetailsHistoryEntry.JobId = jobCreationResults.JobId; jobDetailsHistoryEntry.PercentComplete = "map 0% reduce 0%"; this.cluster.JobQueue.Add(jobDetailsHistoryEntry.JobId, jobDetailsHistoryEntry); return(jobCreationResults); } }
public async Task ProcessRecord() { if (string.IsNullOrWhiteSpace(this.JobId) || string.IsNullOrWhiteSpace(this.Cluster)) { this.JobId = this.Job.JobId; this.Cluster = this.Job.Cluster; } IJobSubmissionClient client = this.GetClient(this.Cluster); client.JobStatusEvent += this.ClientOnJobStatus; JobDetails jobDetail = null; if (ReduceWaitTime) { jobDetail = await client.GetJobAsync(this.JobId); while (jobDetail.StatusCode != JobStatusCode.Completed && jobDetail.StatusCode != JobStatusCode.Failed && jobDetail.StatusCode != JobStatusCode.Canceled) { jobDetail = await client.GetJobAsync(this.JobId); } } else { var job = new JobCreationResults { JobId = this.JobId }; jobDetail = await client.WaitForJobCompletionAsync(job, TimeSpan.FromSeconds(this.WaitTimeoutInSeconds), this.tokenSource.Token); } this.Output.Add(new AzureHDInsightJob(jobDetail, this.Cluster)); }
/// <summary> /// Deserializes a jobDetails creation result object from a payload string. /// </summary> /// <param name="payload"> /// The payload. /// </param> /// <returns> /// A JobCreationResults object representing the payload. /// </returns> public JobCreationResults DeserializeJobCreationResults(string payload) { JobCreationResults results = new JobCreationResults(); results.ErrorCode = string.Empty; results.HttpStatusCode = HttpStatusCode.Accepted; XmlDocumentConverter documentConverter = new XmlDocumentConverter(); var document = documentConverter.GetXmlDocument(payload); DynaXmlNamespaceTable nameTable = new DynaXmlNamespaceTable(document); var node = document.SelectSingleNode("//def:PassthroughResponse/def:Data", nameTable.NamespaceManager); results.JobId = node.InnerText; XmlElement error = (XmlElement)document.SelectSingleNode("//def:PassthroughResponse/def:Error", nameTable.NamespaceManager); if (error.IsNotNull()) { var errorId = error.SelectSingleNode("//def:ErrorId", nameTable.NamespaceManager); if (errorId.IsNotNull()) { results.ErrorCode = errorId.InnerText; } var statusCode = error.SelectSingleNode("//def:StatusCode", nameTable.NamespaceManager); if (statusCode.IsNotNull()) { HttpStatusCode httpStatusCode = HttpStatusCode.Accepted; if (HttpStatusCode.TryParse(statusCode.InnerText, out httpStatusCode)) { results.HttpStatusCode = httpStatusCode; } } } return(results); }
public Task <JobCreationResults> CreateHiveJobAsync(HiveJobCreateParameters hiveJobCreateParameters) { if (hiveJobCreateParameters.Query.IsNullOrEmpty()) { hiveJobCreateParameters.File.ArgumentNotNullOrEmpty("File"); if (hiveJobCreateParameters.File.Contains("://") && !hiveJobCreateParameters.File.StartsWith("wasb", StringComparison.OrdinalIgnoreCase)) { throw new InvalidOperationException("Invalid file protocol : " + hiveJobCreateParameters.File); } } else { this.PrepareQueryJob(hiveJobCreateParameters); } JobCreationResults retval = this.CreateJobSuccessResult( new JobDetails { Name = hiveJobCreateParameters.JobName, Query = hiveJobCreateParameters.Query, StatusDirectory = hiveJobCreateParameters.StatusFolder }, hiveJobCreateParameters.JobName); return(TaskEx2.FromResult(retval)); }
//Run Hive Job public static void DoHiveOperations() { HiveJobCreateParameters hiveJobDefinition = new HiveJobCreateParameters() { JobName = "Show tables job", StatusFolder = "/TableListFolder", Query = "show tables;" }; var store = new X509Store(); store.Open(OpenFlags.ReadOnly); var cert = store.Certificates.Cast <X509Certificate2>().First(item => item.Thumbprint == Constants.thumbprint); var creds = new JobSubmissionCertificateCredential(Constants.subscriptionId, cert, Constants.clusterName); var jobClient = JobSubmissionClientFactory.Connect(creds); JobCreationResults jobResults = jobClient.CreateHiveJob(hiveJobDefinition); Console.Write("Executing Hive Job."); // Wait for the job to complete WaitForJobCompletion(jobResults, jobClient); // Print the Hive job output System.IO.Stream stream = jobClient.GetJobOutput(jobResults.JobId); System.IO.StreamReader reader = new System.IO.StreamReader(stream); Console.Write("Done..List of Tables are:\n"); Console.WriteLine(reader.ReadToEnd()); }
/// <summary> /// Performs HQL query and returns the query results. /// </summary> /// <param name="jobParams">The query parameters.</param> /// <returns>The query result.</returns> public string Query(HiveJobCreateParameters jobParams) { // Assign status folder jobParams.StatusFolder = RootDirectory + "/status"; JobCreationResults jobDetails = null; try { // Create Hive job jobDetails = this.job.CreateHiveJob(jobParams); } catch (Exception e) { AvroHdiSample.ReportError("Error while creating a Hive job\n" + e); } JobDetails jobInProgress = null; try { // Get job status jobInProgress = this.job.GetJob(jobDetails.JobId); } catch (Exception e) { AvroHdiSample.ReportError("Error while getting Hive job status\n" + e); } // If job is not finished then sleep until the next client polling interval while (jobInProgress.StatusCode != JobStatusCode.Completed && jobInProgress.StatusCode != JobStatusCode.Failed) { try { // Get job status jobInProgress = this.job.GetJob(jobDetails.JobId); } catch (Exception e) { AvroHdiSample.ReportError("Error while getting Hive job status\n" + e); } Thread.Sleep(this.client.PollingInterval); } try { // Job is finished; get its output stream, read it, and return the value return(new StreamReader(this.job.GetJobOutput(jobDetails.JobId)).ReadToEnd()); } catch (Exception e) { AvroHdiSample.ReportError("Error while reading Hibe job result\n" + e); } return(string.Empty); }
public void ServiceHost_JobSubmissionRecieved(object sender, JobSubmissionMessage e) { Trace.WriteLine("JobSubmissionRecieved Recieved User Id : " + e.idUsuario, "Warning"); try { // Obtener la cuenta de almacenamiento // Para actualizar metadatos CloudStorageAccount storageAccount = CloudStorageAccount.Parse( CloudConfigurationManager.GetSetting("StorageConnectionString")); CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient(); CloudBlobContainer container = blobClient.GetContainerReference(VariablesConfiguracion.containerName); // Obtencion de variables para la conxion con el cluster string subscriptionID = VariablesConfiguracion.subscriptionID; string certFriendlyName = VariablesConfiguracion.certFriendlyName; string clusterName = VariablesConfiguracion.clusterName; // Definicion de la tarea MapReduce MapReduceJobCreateParameters mrJobDefinition = new MapReduceJobCreateParameters() { JarFile = "wasb:///CienciaCelularMR.jar", ClassName = "Main", StatusFolder = "wasb:///scicluster/test/status-" + e.idUsuario + "." + e.subIdUsuario, }; mrJobDefinition.Arguments.Add("wasb:///" + e.nomEntrada); mrJobDefinition.Arguments.Add("wasb:///scicluster/test/output-" + e.idUsuario + "." + e.subIdUsuario); // Obtener el objeto certificate X509Store store = new X509Store(); store.Open(OpenFlags.ReadOnly); X509Certificate2 cert = FindCertificate(StoreLocation.CurrentUser, StoreName.My, X509FindType.FindByThumbprint, VariablesConfiguracion.thumbprint); JobSubmissionCertificateCredential creds = new JobSubmissionCertificateCredential(new Guid(subscriptionID), cert, clusterName); // Se crea un cliente Hadoop para conectarse con HDInsight var jobClient = JobSubmissionClientFactory.Connect(creds); //Actualizacion de metadatos CloudBlockBlob infoSimulation = container.GetBlockBlobReference(VariablesConfiguracion.infoSimulation + "-" + e.idUsuario); infoSimulation.UploadText(VariablesConfiguracion.JOB_STARTING); // Se lanza la ejecucion de los jobs MapReduce JobCreationResults mrJobResults = jobClient.CreateMapReduceJob(mrJobDefinition); // Esperar hasta que finalice la ejecucion WaitForJobCompletion(mrJobResults, jobClient, e.idUsuario, e.subIdUsuario); } catch (Exception ex) { Trace.TraceError(ex.Message); throw; } }
private static void WaitForJobCompletion(IJobSubmissionClient client, JobCreationResults jobResults) { JobDetails jobInProgress = client.GetJob(jobResults.JobId); while (jobInProgress.StatusCode != JobStatusCode.Completed && jobInProgress.StatusCode != JobStatusCode.Failed) { jobInProgress = client.GetJob(jobInProgress.JobId); Thread.Sleep(TimeSpan.FromSeconds(10)); } }
public Task <JobCreationResults> SubmitHiveJob(HiveJobCreateParameters details) { this.SubmitHiveJobCalled = true; var job = new JobCreationResults() { JobId = JobId }; return(Task.Run(() => job)); }
public Task <JobCreationResults> SubmitSqoopJob(SqoopJobCreateParameters sqoopJobCreateParameters) { this.SubmitSqoopJobCalled = true; var job = new JobCreationResults() { JobId = JobId }; return(Task.Run(() => job)); }
public void ICanSerializeAndDeserialzeCreationResults() { JobCreationResults expected = new JobCreationResults() { HttpStatusCode = HttpStatusCode.Accepted, JobId = "job123" }; JobPayloadServerConverter ser = new JobPayloadServerConverter(); JobPayloadConverter deser = new JobPayloadConverter(); var payload = ser.SerializeJobCreationResults(expected); var actual = deser.DeserializeJobCreationResults(payload); }
internal AzureYarnJob(AzureYarnClient client, Uri errorLocation, Uri outputDir, JobCreationResults launchJob) { this.client = client; this.errorLocation = errorLocation; this.outputDir = outputDir; this.launchJob = launchJob; this.status = JobStatus.Waiting; this.completion = new TaskCompletionSource <JobStatus>(); this.started = new TaskCompletionSource <bool>(); Task.Run(() => MonitorJob()); }
public string SerializeJobCreationResults(JobCreationResults jobResults) { var result = new PassthroughResponse(); if (jobResults.ErrorCode.IsNotNullOrEmpty() || jobResults.HttpStatusCode != HttpStatusCode.Accepted) { result.Error = new PassthroughErrorResponse { StatusCode = jobResults.HttpStatusCode, ErrorId = jobResults.ErrorCode }; } result.Data = jobResults.JobId; return(this.SerializeJobDetails(result)); }
private static JobCreationResults RunHiveJob(HiveJobCreateParameters job) { JobCreationResults jobDetails = jobSubmissionClient.CreateHiveJob(job); JobDetails jobInProgress = jobSubmissionClient.GetJob(jobDetails.JobId); while (jobInProgress.StatusCode != JobStatusCode.Completed && jobInProgress.StatusCode != JobStatusCode.Failed) { jobInProgress = jobSubmissionClient.GetJob(jobInProgress.JobId); Thread.Sleep(TimeSpan.FromMilliseconds(IHadoopClientExtensions.GetPollingInterval())); } Assert.IsNull(jobDetails.ErrorCode, "Should not fail hive jobDetails submission"); Assert.IsNotNull(jobDetails.JobId, "Should have a non-null jobDetails id"); return(jobDetails); }
private static void WaitForJobCompletion(JobCreationResults jobResults, IJobSubmissionClient client) { // Wait for job completion, displaying progress JobDetails jobInProgress = client.GetJob(jobResults.JobId); Console.Write(jobResults.JobId); while (jobInProgress.StatusCode != JobStatusCode.Completed && jobInProgress.StatusCode != JobStatusCode.Failed) { jobInProgress = client.GetJob(jobInProgress.JobId); Thread.Sleep(TimeSpan.FromSeconds(5)); Console.Write("."); } Console.WriteLine(jobInProgress.StatusCode); }
public Task <JobCreationResults> CreateSqoopJobAsync(SqoopJobCreateParameters sqoopJobCreateParameters) { if (sqoopJobCreateParameters == null) { throw new ArgumentNullException("sqoopJobCreateParameters"); } JobCreationResults retval = this.CreateJobSuccessResult( new JobDetails { Query = sqoopJobCreateParameters.Command, StatusDirectory = sqoopJobCreateParameters.StatusFolder }, string.Empty); return(TaskEx2.FromResult(retval)); }
//Helper Function to Wait while job executes private static void WaitForJobCompletion(JobCreationResults jobResults, IJobSubmissionClient client) { Trace.WriteLine("Entering WaitForJobCompletion method"); Trace.TraceInformation("Executing WaitForJobCompletion method " + DateTime.Now.ToLongTimeString()); JobDetails jobInProgress = client.GetJob(jobResults.JobId); while (jobInProgress.StatusCode != JobStatusCode.Completed && jobInProgress.StatusCode != JobStatusCode.Failed) { jobInProgress = client.GetJob(jobInProgress.JobId); Thread.Sleep(TimeSpan.FromSeconds(1)); Console.Write("."); } Trace.WriteLine("Leaving WaitForJobCompletion method"); }
public Task <JobCreationResults> CreatePigJobAsync(PigJobCreateParameters pigJobCreateParameters) { if (pigJobCreateParameters == null) { throw new ArgumentNullException("pigJobCreateParameters"); } this.PrepareQueryJob(pigJobCreateParameters); JobCreationResults retval = this.CreateJobSuccessResult( new JobDetails { Query = pigJobCreateParameters.Query, StatusDirectory = pigJobCreateParameters.StatusFolder }, string.Empty); return(TaskEx2.FromResult(retval)); }
private void RoundtripAvroDataToHive() { this.UploadFiles(); string testDirectory = string.Format( "{0}{1}@{2}/{3}", Constants.WabsProtocolSchemeName, this.storageAccount.Container, this.storageAccount.Name, AvroIntegrationWithHiveConfigurations.TestRootDirectory); RunHiveJob(new HiveJobCreateParameters { File = testDirectory + "/query.hql", JobName = "CreateLargeClassTable" }); JobCreationResults result = RunHiveJob( new HiveJobCreateParameters { Query = "SELECT BoolMember, DateTimeMember, DecimalMember, DoubleMember, EnumMember, FloatMember, base64(GuidMember), IntArrayMember, IntListMember, IntMapMember, IntMember, LongMember, SByteArrayMember, StringMember FROM " + AvroIntegrationWithHiveConfigurations.HiveTableName, JobName = "QueryLargeClassTable", StatusFolder = "/" + AvroIntegrationWithHiveConfigurations.QueryResultFolder }); Stream stream = jobSubmissionClient.GetJobOutput(result.JobId); this.dataProvider.CompareToQueryResult(new StreamReader(stream).ReadToEnd()); RunHiveJob(new HiveJobCreateParameters { File = testDirectory + "/extraquery.hql", JobName = "CreateExtraLargeClassTable" }); RunHiveJob( new HiveJobCreateParameters { Query = "INSERT INTO TABLE " + AvroIntegrationWithHiveConfigurations.HiveExtraTableName + " SELECT * FROM " + AvroIntegrationWithHiveConfigurations.HiveTableName + ";" }); Task <Stream> task = this.storageClient.Read(new Uri(testDirectory + "/extradata/000000_0")); task.Wait(); Stream streamResult = task.Result; this.dataProvider.CompareToAvroFile(streamResult); }
//Run Sample Map Reduce Job public static void DoMapReduce() { // Define the MapReduce job MapReduceJobCreateParameters mrJobDefinition = new MapReduceJobCreateParameters() { JarFile = "wasb:///example/jars/hadoop-examples.jar", ClassName = "wordcount" }; mrJobDefinition.Arguments.Add("wasb:///example/data/gutenberg/davinci.txt"); mrJobDefinition.Arguments.Add("wasb:///example/data/WordCountOutput"); //Get certificate var store = new X509Store(); store.Open(OpenFlags.ReadOnly); var cert = store.Certificates.Cast <X509Certificate2>().First(item => item.Thumbprint == Constants.thumbprint); var creds = new JobSubmissionCertificateCredential(Constants.subscriptionId, cert, Constants.clusterName); // Create a hadoop client to connect to HDInsight var jobClient = JobSubmissionClientFactory.Connect(creds); // Run the MapReduce job JobCreationResults mrJobResults = jobClient.CreateMapReduceJob(mrJobDefinition); Console.Write("Executing WordCount MapReduce Job."); // Wait for the job to complete WaitForJobCompletion(mrJobResults, jobClient); // Print the MapReduce job output Stream stream = new MemoryStream(); CloudStorageAccount storageAccount = CloudStorageAccount.Parse("DefaultEndpointsProtocol=https;AccountName=" + Constants.storageAccount + ";AccountKey=" + Constants.storageAccountKey); CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient(); CloudBlobContainer blobContainer = blobClient.GetContainerReference(Constants.container); CloudBlockBlob blockBlob = blobContainer.GetBlockBlobReference("example/data/WordCountOutput/part-r-00000"); blockBlob.DownloadToStream(stream); stream.Position = 0; StreamReader reader = new StreamReader(stream); Console.Write("Done..Word counts are:\n"); Console.WriteLine(reader.ReadToEnd()); }
private static void WaitForJobCompletion(JobCreationResults jobResults, IJobSubmissionClient client, String idUsuario, String subIdUsuario) { try { // Retrieve storage account from connection string. CloudStorageAccount storageAccount = CloudStorageAccount.Parse( CloudConfigurationManager.GetSetting("Microsoft.WindowsAzure.Plugins.Diagnostics.ConnectionString")); // Create the blob client. CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient(); // Retrieve reference to a previously created container. CloudBlobContainer container = blobClient.GetContainerReference(VariablesConfiguracion.containerName); CloudBlockBlob blockBlob = container.GetBlockBlobReference("pruebaProgreso-" + idUsuario + "." + subIdUsuario); JobDetails jobInProgress = client.GetJob(jobResults.JobId); while (jobInProgress.StatusCode != JobStatusCode.Completed && jobInProgress.StatusCode != JobStatusCode.Failed) { blockBlob.UploadText(VariablesConfiguracion.JOB_RUNNING); jobInProgress = client.GetJob(jobInProgress.JobId); } if (jobInProgress.StatusCode == JobStatusCode.Failed) { blockBlob.UploadText(VariablesConfiguracion.JOB_FAIL); } else if (jobInProgress.StatusCode == JobStatusCode.Completed) { blockBlob.UploadText(VariablesConfiguracion.JOB_SUCCESS); } } catch (Exception ex) { Trace.TraceError(ex.Message); throw; } finally { } }
private void InitializeSimulator() { if (this.cluster.IsNull()) { throw new InvalidOperationException("Cluster is not set to a valid instance."); } JobCreationResults piJob = this.CreateMapReduceJob( new MapReduceJobCreateParameters { ClassName = "pi", JarFile = "wasb://containerName@hostname/jarfileName", JobName = "pi estimation jobDetails", StatusFolder = "/pioutput" }); JobCreationResults hiveJob = this.CreateHiveJob( new HiveJobCreateParameters { JobName = "show tables jobDetails", Query = "show tables", StatusFolder = "/hiveoutput" }); JobCreationResults pigJob = this.CreatePigJob(new PigJobCreateParameters { Query = "show tables", StatusFolder = "/pigoutput" }); this.WriteJobOutput(piJob.JobId, "/pioutput", JobSuccesful); this.WriteJobOutput(hiveJob.JobId, "/hiveoutput", "hivesampletable"); this.WriteJobOutput(pigJob.JobId, "/pigoutput", JobSuccesful); this.WriteJobError(piJob.JobId, "/pioutput", JobSuccesful); this.WriteJobError(hiveJob.JobId, "/hiveoutput", JobSuccesful); this.WriteJobError(pigJob.JobId, "/pigoutput", JobSuccesful); this.WriteJobLogSummary("/pioutput", piJob.JobId); this.WriteJobLogSummary("/hiveoutput", hiveJob.JobId); this.WriteJobLogSummary("/pigoutput", pigJob.JobId); }
private static void SubmitJobs() { // Get HDInsight cluster configuration settings string clusterName = ConfigurationManager.AppSettings["ClusterName"]; string userName = ConfigurationManager.AppSettings["UserName"]; string password = ConfigurationManager.AppSettings["Password"]; // Create basic authentication credential for cluster BasicAuthCredential bcred = new BasicAuthCredential(); bcred.Server = new Uri("https://" + clusterName + ".azurehdinsight.net"); bcred.UserName = userName; bcred.Password = password; // Create and submit Pig job PigJobCreateParameters pigJob = new PigJobCreateParameters() { StatusFolder = "/data/racecar/scripts/processdatastatus", File = "/data/racecar/scripts/processdata.pig" }; var pigJobClient = JobSubmissionClientFactory.Connect(bcred); JobCreationResults pigJobResults = pigJobClient.CreatePigJob(pigJob); WaitForJobCompletion(pigJobResults, pigJobClient); // Create and submit Hive job HiveJobCreateParameters hiveJob = new HiveJobCreateParameters() { JobName = "Create Hive tables", StatusFolder = "/data/racecar/scripts/createtablestatus", File = "/data/racecar/scripts/createtables.hql" }; var hiveJobClient = JobSubmissionClientFactory.Connect(bcred); JobCreationResults hiveJobResults = hiveJobClient.CreateHiveJob(hiveJob); WaitForJobCompletion(hiveJobResults, hiveJobClient); }
/// <inheritdoc /> private async Task DownloadApplicationLogsAsync(string applicationId, string applicationUser, string containerId, string nodeId, string targetDirectory) { applicationId.ArgumentNotNullOrEmpty("applicationId"); applicationUser.ArgumentNotNullOrEmpty("applicationUser"); targetDirectory.ArgumentNotNullOrEmpty("targetDirectory"); if (!string.IsNullOrEmpty(containerId) && string.IsNullOrEmpty(nodeId)) { throw new ArgumentException("NodeId was null or empty. If container id is specified, node id should also be specified"); } if (!Directory.Exists(targetDirectory)) { throw new ArgumentException(string.Format(CultureInfo.InvariantCulture, "The specified directory {0} does not exist.", targetDirectory)); } var jobSubmissionClient = JobSubmissionClientFactory.Connect(this.HttpCredentials, this.customUserAgent); var storageClient = ServiceLocator.Instance.Locate <IWabStorageAbstractionFactory>().Create(this.DefaultStorageCredentials); // Check existence of application logs in the default storage account Uri appLogContainer = new Uri(string.Format(CultureInfo.InvariantCulture, "{0}{1}@{2}/app-logs/{3}/logs/{4}", Constants.WabsProtocolSchemeName, this.DefaultStorageCredentials.ContainerName, this.DefaultStorageCredentials.Name, applicationUser, applicationId)); var logFiles = await storageClient.List(appLogContainer, false); if (!logFiles.Any()) { throw new InvalidOperationException(string.Format("No logs found for application id {0}, user {1}, on cluster {2} at location {3}", applicationId, applicationUser, this.Cluster.Name, appLogContainer.AbsoluteUri)); } // Application logs exist! // Convert them to plain text by running YARN CLI string jobName = string.Format("yarnlogs-{0}", Guid.NewGuid()); string statusFolderName = string.Format("/{0}", jobName); string optionalContainerArguments = !string.IsNullOrEmpty(containerId) ? string.Format(" -containerId {0} -nodeAddress {1}", containerId, nodeId) : string.Empty; string command = ""; if (this.Cluster.OSType == OSType.Windows) { command = string.Format("!cmd.exe /c yarn logs -applicationId {0} -appOwner {1}{2};", applicationId, applicationUser, optionalContainerArguments); } else if (this.Cluster.OSType == OSType.Linux) { command = string.Format("!yarn logs -applicationId {0} -appOwner {1}{2};", applicationId, applicationUser, optionalContainerArguments); } else { throw new NotSupportedException(String.Format("This functionality is not supported on clusters with OS Type: {0}", this.Cluster.OSType)); } string queryFileName = string.Format("/{0}.hql", jobName); Uri queryFileUri = new Uri(string.Format(CultureInfo.InvariantCulture, "{0}{1}@{2}{3}", Constants.WabsProtocolSchemeName, this.DefaultStorageCredentials.ContainerName, this.DefaultStorageCredentials.Name, queryFileName)); Uri statusFolderUri = new Uri(string.Format(CultureInfo.InvariantCulture, "{0}{1}@{2}{3}", Constants.WabsProtocolSchemeName, this.DefaultStorageCredentials.ContainerName, this.DefaultStorageCredentials.Name, statusFolderName)); try { var bytes = Encoding.UTF8.GetBytes(command); using (var memoryStream = new MemoryStream(bytes)) { await storageClient.Write(queryFileUri, memoryStream); } HiveJobCreateParameters hiveJobDefinition = new HiveJobCreateParameters() { JobName = jobName, StatusFolder = statusFolderName, File = queryFileName }; JobCreationResults jobResults = jobSubmissionClient.CreateHiveJob(hiveJobDefinition); WaitForJobCompletion(jobSubmissionClient, jobResults); Uri logContentsFileUri = new Uri(string.Format("{0}/stdout", statusFolderUri.AbsoluteUri)); if (await storageClient.Exists(logContentsFileUri)) { // create local file in the targetdirectory. var localFilePath = Path.Combine(targetDirectory, string.Format("{0}_{1}.txt", this.Cluster.Name, string.IsNullOrEmpty(containerId) ? applicationId : containerId)); await storageClient.DownloadToFile(logContentsFileUri, localFilePath); } else { throw new InvalidOperationException(string.Format( CultureInfo.InvariantCulture, "Could not retrive logs for application id {0}, user {1} on cluster {2} at location {3}.", applicationId, applicationUser, this.Cluster.Name, appLogContainer.AbsoluteUri)); } } finally { // Cleanup what we created if (storageClient.Exists(queryFileUri).WaitForResult()) { storageClient.Delete(queryFileUri); } if (storageClient.Exists(statusFolderUri).WaitForResult()) { storageClient.Delete(statusFolderUri); } } }
public string HiveOutput(string q) { Trace.WriteLine("Entering HiveOutput method"); Trace.TraceInformation("Executing HiveOutput method at " + DateTime.Now.ToLongTimeString()); //Defining MapReduce Job HiveJobCreateParameters hiveJobDefinition = new HiveJobCreateParameters() { JobName = "job", StatusFolder = "/TableListFolder", Query = q }; Guid subscriptionId = new Guid("44fbb137-edbb-4044-9db9-0e1333e137cf"); //your-subscription-id string clusterName = "tbanihumcluster"; // Get the certificate object from certificate store using the friendly name to identify it X509Store store = new X509Store(StoreName.My); store.Open(OpenFlags.ReadOnly); X509Certificate2 cert = store.Certificates.Cast <X509Certificate2>().First(item => item.FriendlyName == "Azdem187U23713U-1-8-2015-credentials"); var creds = new JobSubmissionCertificateCredential(subscriptionId, cert, clusterName); // Create a hadoop client to connect to HDInsight var jobClient = JobSubmissionClientFactory.Connect(creds); // Run the MapReduce job JobCreationResults jobResults = jobClient.CreateHiveJob(hiveJobDefinition); // Wait for the job to complete WaitForJobCompletion(jobResults, jobClient); // Hive job output System.IO.Stream stream = jobClient.GetJobOutput(jobResults.JobId); System.IO.StreamReader reader = new System.IO.StreamReader(stream); string value = reader.ReadToEnd(); value = value.Replace('\t', ','); // Retrieve storage account from connection string. CloudStorageAccount storageAccount = CloudStorageAccount.Parse(CloudConfigurationManager.GetSetting("StorageConnectionString")); // Create the blob client. CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient(); // Retrieve reference to a previously created container. CloudBlobContainer container = blobClient.GetContainerReference("mycontainer"); container.CreateIfNotExists(); // Retrieve reference to a blob named "myblob". CloudBlockBlob blockBlob = container.GetBlockBlobReference("myblob"); blockBlob.UploadText(value); Trace.WriteLine("Leaving HiveOutput method"); return(value); }
public override async Task EndProcessing() { this.JobDefinition.ArgumentNotNull("JobDefinition"); IJobSubmissionClient client = this.GetClient(this.Cluster); JobCreationResults jobCreationResults = null; if (this.JobDefinition.StatusFolder.IsNullOrEmpty()) { this.JobDefinition.StatusFolder = Guid.NewGuid().ToString(); } var azureMapReduceJobDefinition = this.JobDefinition as AzureHDInsightMapReduceJobDefinition; var azureHiveJobDefinition = this.JobDefinition as AzureHDInsightHiveJobDefinition; var azurePigJobDefinition = this.JobDefinition as AzureHDInsightPigJobDefinition; var azureStreamingJobDefinition = this.JobDefinition as AzureHDInsightStreamingMapReduceJobDefinition; var azureSqoopJobDefinition = this.JobDefinition as AzureHDInsightSqoopJobDefinition; if (azureMapReduceJobDefinition != null) { if (azureMapReduceJobDefinition.JobName.IsNullOrEmpty()) { azureMapReduceJobDefinition.JobName = azureMapReduceJobDefinition.ClassName; } jobCreationResults = await SubmitMapReduceJob(azureMapReduceJobDefinition, client); } else if (azureHiveJobDefinition != null) { if (azureHiveJobDefinition.JobName.IsNullOrEmpty()) { azureHiveJobDefinition.JobName = string.Format( CultureInfo.InvariantCulture, HiveJobNameFormat, GetJobNameFromQueryOrFile(azureHiveJobDefinition.Query, azureHiveJobDefinition.File)); } jobCreationResults = await SubmitHiveJob(azureHiveJobDefinition, client); } else if (azurePigJobDefinition != null) { jobCreationResults = await SubmitPigJob(azurePigJobDefinition, client); } else if (azureStreamingJobDefinition != null) { if (azureStreamingJobDefinition.JobName.IsNullOrEmpty()) { azureStreamingJobDefinition.JobName = GetLastSegment(azureStreamingJobDefinition.Mapper); } jobCreationResults = await CreateStreamingJob(azureStreamingJobDefinition, client); } else if (azureSqoopJobDefinition != null) { jobCreationResults = await CreateSqoopJob(azureSqoopJobDefinition, client); } else { throw new NotSupportedException( string.Format(CultureInfo.InvariantCulture, "Cannot start jobDetails of type : {0}.", this.JobDefinition.GetType())); } var startedJob = await client.GetJobAsync(jobCreationResults.JobId); if (startedJob.ErrorCode.IsNotNullOrEmpty()) { throw new InvalidOperationException("Failed to start jobDetails :" + startedJob.ErrorCode); } var jobDetail = new AzureHDInsightJob(startedJob, this.Cluster); this.Output.Add(jobDetail); }
private JobCreationResults CreateJobSuccessResult(JobDetails jobDetailsHistoryEntry, string jobName) { if (this.cluster.IsNull()) { throw new InvalidOperationException("The cluster could not be found."); } //if (this.credentials.UserName != this.cluster.Cluster.HttpUserName && this.credentials.Password != this.cluster.Cluster.HttpPassword) //{ // throw new UnauthorizedAccessException("The supplied credential do not have access to the server."); //} lock (this.cluster.JobQueue) { this.LogMessage("Starting jobDetails '{0}'.", jobName); var jobCreationResults = new JobCreationResults { JobId = "job_" + Guid.NewGuid().ToString(), HttpStatusCode = HttpStatusCode.OK }; jobDetailsHistoryEntry.Name = jobName; jobDetailsHistoryEntry.JobId = jobCreationResults.JobId; jobDetailsHistoryEntry.PercentComplete = "map 0% reduce 0%"; this.cluster.JobQueue.Add(jobDetailsHistoryEntry.JobId, jobDetailsHistoryEntry); return jobCreationResults; } }