public AzureHDInsightJob WaitJob() { _clusterName = GetClusterConnection(ResourceGroupName, ClusterName); TimeSpan?duration = null; if (TimeoutInSeconds > 0) { duration = TimeSpan.FromSeconds(TimeoutInSeconds); } TimeSpan?waitInterval = null; if (WaitIntervalInSeconds > 0) { waitInterval = TimeSpan.FromSeconds(WaitIntervalInSeconds); } JobDetailRootJsonObject jobDetail = null; try { jobDetail = HDInsightJobClient.WaitForJobCompletion(JobId, duration, waitInterval).JobDetail; } catch (TimeoutException) { var exceptionMessage = string.Format(CultureInfo.InvariantCulture, "HDInsight job with ID {0} has not completed in {1} seconds. Increase the value of -TimeoutInSeconds or check the job runtime.", JobId, TimeoutInSeconds); throw new CloudException(exceptionMessage); } var jobDetails = new AzureHDInsightJob(jobDetail, HDInsightJobClient.ClusterName); return(jobDetails); }
private void SubmitHiveJob(string query) { var parameters = new HiveJobSubmissionParameters { Query = query }; // submitting the Hive job to the cluster JobSubmissionResponse jobResponse = _hdiJobManagementClient.JobManagement.SubmitHiveJob(parameters); string jobId = jobResponse.JobSubmissionJsonResponse.Id; // wait for job completion JobDetailRootJsonObject jobDetail = _hdiJobManagementClient.JobManagement.GetJob(jobId).JobDetail; while (!jobDetail.Status.JobComplete) { Thread.Sleep(1000); jobDetail = _hdiJobManagementClient.JobManagement.GetJob(jobId).JobDetail; } // get job output Stream output = jobDetail.ExitValue == 0 ? _hdiJobManagementClient.JobManagement.GetJobOutput(jobId, _storageAccess) : _hdiJobManagementClient.JobManagement.GetJobErrorLogs(jobId, _storageAccess); // handle output using (var reader = new StreamReader(output, Encoding.UTF8)) { string value = reader.ReadToEnd(); if (!string.IsNullOrEmpty(value)) { throw new BatchViewCalculationException(value); } } }
/// <summary> /// Initializes a new instance of the AzureHDInsightJob class. /// </summary> /// <param name="jobDetails">The HDInsight jobDetails.</param> /// <param name="cluster">The cluster that the jobDetails was created against.</param> public AzureHDInsightJob(JobDetailRootJsonObject jobDetails, string cluster) { Cluster = cluster.Substring(0, cluster.IndexOf('.') + 1); HttpEndpoint = cluster; State = jobDetails.Status.State; JobId = jobDetails.Id; ParentId = jobDetails.ParentId; PercentComplete = jobDetails.PercentComplete; ExitValue = jobDetails.ExitValue; User = jobDetails.User; Callback = jobDetails.Callback; Completed = jobDetails.Completed; }
/// <summary> /// Initializes a new instance of the AzureHDInsightJob class. /// </summary> /// <param name="jobDetails">The HDInsight jobDetails.</param> /// <param name="cluster">The cluster that the jobDetails was created against.</param> public AzureHDInsightJob(JobDetailRootJsonObject jobDetails, string cluster) { var index = cluster.IndexOf('.'); Cluster = index > -1 ? cluster.Substring(0, index) : cluster; HttpEndpoint = cluster; State = jobDetails.Status.State; JobId = jobDetails.Id; ParentId = jobDetails.ParentId; PercentComplete = jobDetails.PercentComplete; ExitValue = jobDetails.ExitValue; User = jobDetails.User; Callback = jobDetails.Callback; Completed = jobDetails.Completed; StatusFolder = jobDetails.Userargs.Statusdir != null?jobDetails.Userargs.Statusdir.ToString() : string.Empty; }
private static string GetStatusFolder(JobDetailRootJsonObject job) { return(job.Userargs.Statusdir == null ? null : job.Userargs.Statusdir.ToString()); }