public override void ExecuteCmdlet()
{
if (ResourceGroupName == null)
{
ResourceGroupName = GetResourceGroupByAccountName(ClusterName);
}
_clusterName = GetClusterConnection(ResourceGroupName, ClusterName);
if (NumOfJobs > 0)
{
var jobs = HDInsightJobClient.ListJobsAfterJobId(JobId, NumOfJobs).Select(job => new AzureHDInsightJob(job.Detail, HDInsightJobClient.ClusterName));
WriteObject(jobs, true);
}
else if (JobId != null)
{
var job = HDInsightJobClient.GetJob(JobId);
var jobDetails = new AzureHDInsightJob(job.JobDetail, HDInsightJobClient.ClusterName);
WriteObject(jobDetails);
}
else
{
var jobs = HDInsightJobClient.ListJobs().Select(job => job.Id);
WriteObject(jobs, true);
}
}
internal string GetJobError(IStorageAccess storageAccess)
{
var output = HDInsightJobClient.GetJobError(JobId, storageAccess);
var outputStr = Convert(output);
return(outputStr);
}
internal string GetJobOutput()
{
var output = HDInsightJobClient.GetJobOutput(JobId, DefaultStorageAccountName, DefaultStorageAccountKey, DefaultContainer);
var outputStr = Convert(output);
return(outputStr);
}
public AzureHDInsightJob WaitJob()
{
_clusterName = GetClusterConnection(ResourceGroupName, ClusterName);
TimeSpan?duration = null;
if (TimeoutInSeconds > 0)
{
duration = TimeSpan.FromSeconds(TimeoutInSeconds);
}
TimeSpan?waitInterval = null;
if (WaitIntervalInSeconds > 0)
{
waitInterval = TimeSpan.FromSeconds(WaitIntervalInSeconds);
}
JobDetailRootJsonObject jobDetail = null;
try
{
jobDetail = HDInsightJobClient.WaitForJobCompletion(JobId, duration, waitInterval).JobDetail;
}
catch (TimeoutException)
{
var exceptionMessage = string.Format(CultureInfo.InvariantCulture, "HDInsight job with ID {0} has not completed in {1} seconds. Increase the value of -TimeoutInSeconds or check the job runtime.", JobId, TimeoutInSeconds);
throw new CloudException(exceptionMessage);
}
var jobDetails = new AzureHDInsightJob(jobDetail, HDInsightJobClient.ClusterName);
return(jobDetails);
}
private string GetJobTaskLogSummary()
{
var output = HDInsightJobClient.GetJobTaskLogSummary(JobId, DefaultStorageAccountName, DefaultStorageAccountKey,
DefaultContainer);
var outputStr = Convert(output);
return(outputStr);
}
public override void ExecuteCmdlet()
{
if (ResourceGroupName == null)
{
ResourceGroupName = GetResourceGroupByAccountName(ClusterName);
}
_clusterName = GetClusterConnection(ResourceGroupName, ClusterName);
HDInsightJobClient.StopJob(JobId);
}
protected override void ProcessRecord()
{
if (ResourceGroupName == null)
{
ResourceGroupName = GetResourceGroupByAccountName(ClusterName);
}
_clusterName = GetClusterConnection(ResourceGroupName, ClusterName);
HDInsightJobClient.StopJob(JobId);
}
public AzureHDInsightJob Execute()
{
_clusterName = GetClusterConnection(ResourceGroupName, ClusterName);
var jobCreationResults = SubmitJob();
var startedJob = HDInsightJobClient.GetJob(jobCreationResults.JobSubmissionJsonResponse.Id);
var jobDetail = new AzureHDInsightJob(startedJob.JobDetail, HDInsightJobClient.ClusterName);
return(jobDetail);
}
public AzureHDInsightJob WaitJob()
{
_clusterName = GetClusterConnection(ResourceGroupName, ClusterName);
var jobDetail = HDInsightJobClient.GetJob(JobId).JobDetail;
while (!jobDetail.Status.JobComplete)
{
jobDetail = HDInsightJobClient.GetJob(JobId).JobDetail;
}
var jobDetails = new AzureHDInsightJob(jobDetail, HDInsightJobClient.ClusterName);
return(jobDetails);
}
public override void ExecuteCmdlet()
{
_clusterName = GetClusterConnection(ResourceGroupName, ClusterName);
if (JobId != null)
{
var job = HDInsightJobClient.GetJob(JobId);
var jobDetails = new AzureHDInsightJob(job.JobDetail, HDInsightJobClient.ClusterName);
WriteObject(jobDetails);
}
else
{
var jobs = HDInsightJobClient.ListJobs().Select(job => job.Id);
WriteObject(jobs, true);
}
}
public void CheckValidJobUserName()
{
var credentials = new BasicAuthenticationCredentials
{
UserName = "******",
Password = ""
};
var client = new HDInsightJobClient("TestCluster", credentials);
Assert.Equal(credentials.UserName.ToLower(CultureInfo.CurrentCulture), client.Username);
client = new HDInsightJobClient("TestCluster", credentials, new HttpClient());
Assert.Equal(credentials.UserName.ToLower(CultureInfo.CurrentCulture), client.Username);
}
protected override void ProcessRecord()
{
if (ResourceGroupName == null)
{
ResourceGroupName = GetResourceGroupByAccountName(ClusterName);
}
_clusterName = GetClusterConnection(ResourceGroupName, ClusterName);
if (JobId != null)
{
var job = HDInsightJobClient.GetJob(JobId);
var jobDetails = new AzureHDInsightJob(job.JobDetail, HDInsightJobClient.ClusterName);
WriteObject(jobDetails);
}
else
{
var jobs = HDInsightJobClient.ListJobs().Select(job => job.Id);
WriteObject(jobs, true);
}
}
public JobSubmissionResponse SubmitJob()
{
if (string.IsNullOrEmpty(JobDefinition.StatusFolder))
{
JobDefinition.StatusFolder = Guid.NewGuid().ToString();
}
JobSubmissionResponse jobCreationResults;
var azureMapReduceJobDefinition = JobDefinition as AzureHDInsightMapReduceJobDefinition;
var azureHiveJobDefinition = JobDefinition as AzureHDInsightHiveJobDefinition;
var azurePigJobDefinition = JobDefinition as AzureHDInsightPigJobDefinition;
var azureStreamingJobDefinition = JobDefinition as AzureHDInsightStreamingMapReduceJobDefinition;
var azureSqoopJobDefinition = JobDefinition as AzureHDInsightSqoopJobDefinition;
if (azureMapReduceJobDefinition != null)
{
jobCreationResults = HDInsightJobClient.SubmitMRJob(azureMapReduceJobDefinition);
}
else if (azureHiveJobDefinition != null)
{
jobCreationResults = HDInsightJobClient.SubmitHiveJob(azureHiveJobDefinition);
}
else if (azurePigJobDefinition != null)
{
jobCreationResults = HDInsightJobClient.SubmitPigJob(azurePigJobDefinition);
}
else if (azureStreamingJobDefinition != null)
{
jobCreationResults = HDInsightJobClient.SubmitStreamingJob(azureStreamingJobDefinition);
}
else if (azureSqoopJobDefinition != null)
{
jobCreationResults = HDInsightJobClient.SubmitSqoopJob(azureSqoopJobDefinition);
}
else
{
throw new NotSupportedException(
string.Format(CultureInfo.InvariantCulture, "Cannot start jobDetails of type : {0}.", JobDefinition.GetType()));
}
return(jobCreationResults);
}
public override void ExecuteCmdlet()
{
//get variables from session
var clusterConnection = SessionState.PSVariable.Get(UseAzureHDInsightClusterCommand.ClusterEndpoint).Value.ToString();
var clusterCred =
(PSCredential)SessionState.PSVariable.Get(UseAzureHDInsightClusterCommand.ClusterCred).Value;
var resourceGroup =
SessionState.PSVariable.Get(UseAzureHDInsightClusterCommand.CurrentResourceGroup).Value.ToString();
_clusterName = clusterConnection;
_credential = new BasicAuthenticationCloudCredentials
{
Username = clusterCred.UserName,
Password = clusterCred.Password.ConvertToString()
};
if (clusterConnection == null || clusterCred == null)
{
throw new NullReferenceException(
string.Format(
"The cluster or resource group specified is null. Please use the Use-AzureHDInsightCluster command to connect to a cluster."));
}
//get hive job
var hivejob = hiveJobDefinitionCommand.GetHiveJob();
//start hive job
WriteObject("Submitting hive query...");
var startJobCommand = new StartAzureHDInsightJobCommand
{
ClusterName = clusterConnection,
ResourceGroupName = resourceGroup,
JobDefinition = hivejob,
ClusterCredential = clusterCred
};
var jobCreationResult = startJobCommand.SubmitJob();
var jobid = jobCreationResult.JobSubmissionJsonResponse.Id;
WriteObject(string.Format("Submitted Hive query with jobDetails Id : {0}", jobid));
//wait for job to complete
WriteProgress(new ProgressRecord(0, "Waiting for job to complete", "In Progress"));
var waitJobCommand = new WaitAzureHDInsightJobCommand
{
ClusterCredential = clusterCred,
ResourceGroupName = resourceGroup,
ClusterName = clusterConnection,
JobId = jobid
};
var job = waitJobCommand.WaitJob();
string output;
if (job.ExitValue == 0)
{
//get job output
var getOutputCommand = new GetAzureHDInsightJobOutputCommand
{
ClusterCredential = clusterCred,
ResourceGroupName = resourceGroup,
ClusterName = clusterConnection,
DefaultContainer = DefaultContainer,
DefaultStorageAccountName = DefaultStorageAccountName,
DefaultStorageAccountKey = DefaultStorageAccountKey,
JobId = jobid
};
output = getOutputCommand.GetJobOutput();
}
else
{
//get job error
output = Convert(HDInsightJobClient.GetJobError(jobid, DefaultStorageAccountName, DefaultStorageAccountKey,
DefaultContainer));
}
WriteObject(output);
}
public override void ExecuteCmdlet()
{
_clusterName = GetClusterConnection(ResourceGroupName, ClusterName);
HDInsightJobClient.StopJob(JobId);
}
private void DownloadJobTaskLogs()
{
HDInsightJobClient.GetJobTaskLogSummary(JobId, DefaultStorageAccountName, DefaultStorageAccountKey,
DefaultContainer);
}
