public override void ExecuteCmdlet() { if (ResourceGroupName == null) { ResourceGroupName = GetResourceGroupByAccountName(ClusterName); } _clusterName = GetClusterConnection(ResourceGroupName, ClusterName); if (NumOfJobs > 0) { var jobs = HDInsightJobClient.ListJobsAfterJobId(JobId, NumOfJobs).Select(job => new AzureHDInsightJob(job.Detail, HDInsightJobClient.ClusterName)); WriteObject(jobs, true); } else if (JobId != null) { var job = HDInsightJobClient.GetJob(JobId); var jobDetails = new AzureHDInsightJob(job.JobDetail, HDInsightJobClient.ClusterName); WriteObject(jobDetails); } else { var jobs = HDInsightJobClient.ListJobs().Select(job => job.Id); WriteObject(jobs, true); } }
internal string GetJobError(IStorageAccess storageAccess) { var output = HDInsightJobClient.GetJobError(JobId, storageAccess); var outputStr = Convert(output); return(outputStr); }
internal string GetJobOutput() { var output = HDInsightJobClient.GetJobOutput(JobId, DefaultStorageAccountName, DefaultStorageAccountKey, DefaultContainer); var outputStr = Convert(output); return(outputStr); }
public AzureHDInsightJob WaitJob() { _clusterName = GetClusterConnection(ResourceGroupName, ClusterName); TimeSpan?duration = null; if (TimeoutInSeconds > 0) { duration = TimeSpan.FromSeconds(TimeoutInSeconds); } TimeSpan?waitInterval = null; if (WaitIntervalInSeconds > 0) { waitInterval = TimeSpan.FromSeconds(WaitIntervalInSeconds); } JobDetailRootJsonObject jobDetail = null; try { jobDetail = HDInsightJobClient.WaitForJobCompletion(JobId, duration, waitInterval).JobDetail; } catch (TimeoutException) { var exceptionMessage = string.Format(CultureInfo.InvariantCulture, "HDInsight job with ID {0} has not completed in {1} seconds. Increase the value of -TimeoutInSeconds or check the job runtime.", JobId, TimeoutInSeconds); throw new CloudException(exceptionMessage); } var jobDetails = new AzureHDInsightJob(jobDetail, HDInsightJobClient.ClusterName); return(jobDetails); }
private string GetJobTaskLogSummary() { var output = HDInsightJobClient.GetJobTaskLogSummary(JobId, DefaultStorageAccountName, DefaultStorageAccountKey, DefaultContainer); var outputStr = Convert(output); return(outputStr); }
public override void ExecuteCmdlet() { if (ResourceGroupName == null) { ResourceGroupName = GetResourceGroupByAccountName(ClusterName); } _clusterName = GetClusterConnection(ResourceGroupName, ClusterName); HDInsightJobClient.StopJob(JobId); }
protected override void ProcessRecord() { if (ResourceGroupName == null) { ResourceGroupName = GetResourceGroupByAccountName(ClusterName); } _clusterName = GetClusterConnection(ResourceGroupName, ClusterName); HDInsightJobClient.StopJob(JobId); }
public AzureHDInsightJob Execute() { _clusterName = GetClusterConnection(ResourceGroupName, ClusterName); var jobCreationResults = SubmitJob(); var startedJob = HDInsightJobClient.GetJob(jobCreationResults.JobSubmissionJsonResponse.Id); var jobDetail = new AzureHDInsightJob(startedJob.JobDetail, HDInsightJobClient.ClusterName); return(jobDetail); }
public AzureHDInsightJob WaitJob() { _clusterName = GetClusterConnection(ResourceGroupName, ClusterName); var jobDetail = HDInsightJobClient.GetJob(JobId).JobDetail; while (!jobDetail.Status.JobComplete) { jobDetail = HDInsightJobClient.GetJob(JobId).JobDetail; } var jobDetails = new AzureHDInsightJob(jobDetail, HDInsightJobClient.ClusterName); return(jobDetails); }
public override void ExecuteCmdlet() { _clusterName = GetClusterConnection(ResourceGroupName, ClusterName); if (JobId != null) { var job = HDInsightJobClient.GetJob(JobId); var jobDetails = new AzureHDInsightJob(job.JobDetail, HDInsightJobClient.ClusterName); WriteObject(jobDetails); } else { var jobs = HDInsightJobClient.ListJobs().Select(job => job.Id); WriteObject(jobs, true); } }
public void CheckValidJobUserName() { var credentials = new BasicAuthenticationCredentials { UserName = "******", Password = "" }; var client = new HDInsightJobClient("TestCluster", credentials); Assert.Equal(credentials.UserName.ToLower(CultureInfo.CurrentCulture), client.Username); client = new HDInsightJobClient("TestCluster", credentials, new HttpClient()); Assert.Equal(credentials.UserName.ToLower(CultureInfo.CurrentCulture), client.Username); }
protected override void ProcessRecord() { if (ResourceGroupName == null) { ResourceGroupName = GetResourceGroupByAccountName(ClusterName); } _clusterName = GetClusterConnection(ResourceGroupName, ClusterName); if (JobId != null) { var job = HDInsightJobClient.GetJob(JobId); var jobDetails = new AzureHDInsightJob(job.JobDetail, HDInsightJobClient.ClusterName); WriteObject(jobDetails); } else { var jobs = HDInsightJobClient.ListJobs().Select(job => job.Id); WriteObject(jobs, true); } }
public JobSubmissionResponse SubmitJob() { if (string.IsNullOrEmpty(JobDefinition.StatusFolder)) { JobDefinition.StatusFolder = Guid.NewGuid().ToString(); } JobSubmissionResponse jobCreationResults; var azureMapReduceJobDefinition = JobDefinition as AzureHDInsightMapReduceJobDefinition; var azureHiveJobDefinition = JobDefinition as AzureHDInsightHiveJobDefinition; var azurePigJobDefinition = JobDefinition as AzureHDInsightPigJobDefinition; var azureStreamingJobDefinition = JobDefinition as AzureHDInsightStreamingMapReduceJobDefinition; var azureSqoopJobDefinition = JobDefinition as AzureHDInsightSqoopJobDefinition; if (azureMapReduceJobDefinition != null) { jobCreationResults = HDInsightJobClient.SubmitMRJob(azureMapReduceJobDefinition); } else if (azureHiveJobDefinition != null) { jobCreationResults = HDInsightJobClient.SubmitHiveJob(azureHiveJobDefinition); } else if (azurePigJobDefinition != null) { jobCreationResults = HDInsightJobClient.SubmitPigJob(azurePigJobDefinition); } else if (azureStreamingJobDefinition != null) { jobCreationResults = HDInsightJobClient.SubmitStreamingJob(azureStreamingJobDefinition); } else if (azureSqoopJobDefinition != null) { jobCreationResults = HDInsightJobClient.SubmitSqoopJob(azureSqoopJobDefinition); } else { throw new NotSupportedException( string.Format(CultureInfo.InvariantCulture, "Cannot start jobDetails of type : {0}.", JobDefinition.GetType())); } return(jobCreationResults); }
public override void ExecuteCmdlet() { //get variables from session var clusterConnection = SessionState.PSVariable.Get(UseAzureHDInsightClusterCommand.ClusterEndpoint).Value.ToString(); var clusterCred = (PSCredential)SessionState.PSVariable.Get(UseAzureHDInsightClusterCommand.ClusterCred).Value; var resourceGroup = SessionState.PSVariable.Get(UseAzureHDInsightClusterCommand.CurrentResourceGroup).Value.ToString(); _clusterName = clusterConnection; _credential = new BasicAuthenticationCloudCredentials { Username = clusterCred.UserName, Password = clusterCred.Password.ConvertToString() }; if (clusterConnection == null || clusterCred == null) { throw new NullReferenceException( string.Format( "The cluster or resource group specified is null. Please use the Use-AzureHDInsightCluster command to connect to a cluster.")); } //get hive job var hivejob = hiveJobDefinitionCommand.GetHiveJob(); //start hive job WriteObject("Submitting hive query..."); var startJobCommand = new StartAzureHDInsightJobCommand { ClusterName = clusterConnection, ResourceGroupName = resourceGroup, JobDefinition = hivejob, ClusterCredential = clusterCred }; var jobCreationResult = startJobCommand.SubmitJob(); var jobid = jobCreationResult.JobSubmissionJsonResponse.Id; WriteObject(string.Format("Submitted Hive query with jobDetails Id : {0}", jobid)); //wait for job to complete WriteProgress(new ProgressRecord(0, "Waiting for job to complete", "In Progress")); var waitJobCommand = new WaitAzureHDInsightJobCommand { ClusterCredential = clusterCred, ResourceGroupName = resourceGroup, ClusterName = clusterConnection, JobId = jobid }; var job = waitJobCommand.WaitJob(); string output; if (job.ExitValue == 0) { //get job output var getOutputCommand = new GetAzureHDInsightJobOutputCommand { ClusterCredential = clusterCred, ResourceGroupName = resourceGroup, ClusterName = clusterConnection, DefaultContainer = DefaultContainer, DefaultStorageAccountName = DefaultStorageAccountName, DefaultStorageAccountKey = DefaultStorageAccountKey, JobId = jobid }; output = getOutputCommand.GetJobOutput(); } else { //get job error output = Convert(HDInsightJobClient.GetJobError(jobid, DefaultStorageAccountName, DefaultStorageAccountKey, DefaultContainer)); } WriteObject(output); }
public override void ExecuteCmdlet() { _clusterName = GetClusterConnection(ResourceGroupName, ClusterName); HDInsightJobClient.StopJob(JobId); }
private void DownloadJobTaskLogs() { HDInsightJobClient.GetJobTaskLogSummary(JobId, DefaultStorageAccountName, DefaultStorageAccountKey, DefaultContainer); }