public async Task ProcessRecord() { if (string.IsNullOrWhiteSpace(this.JobId) || string.IsNullOrWhiteSpace(this.Cluster)) { this.JobId = this.Job.JobId; this.Cluster = this.Job.Cluster; } IJobSubmissionClient client = this.GetClient(this.Cluster); client.JobStatusEvent += this.ClientOnJobStatus; JobDetails jobDetail = null; if (ReduceWaitTime) { jobDetail = await client.GetJobAsync(this.JobId); while (jobDetail.StatusCode != JobStatusCode.Completed && jobDetail.StatusCode != JobStatusCode.Failed && jobDetail.StatusCode != JobStatusCode.Canceled) { jobDetail = await client.GetJobAsync(this.JobId); } } else { var job = new JobCreationResults { JobId = this.JobId }; jobDetail = await client.WaitForJobCompletionAsync(job, TimeSpan.FromSeconds(this.WaitTimeoutInSeconds), this.tokenSource.Token); } this.Output.Add(new AzureHDInsightJob(jobDetail, this.Cluster)); }
private static async Task <JobDetails> GetJobWithRetry(IJobSubmissionClient client, string jobId, CancellationToken cancellationToken) { JobDetails jobDetailsResults = null; var pollingInterval = GetPollingInterval(); int retryCount = 0; while (jobDetailsResults.IsNull()) { try { jobDetailsResults = await client.GetJobAsync(jobId); break; } catch (HttpLayerException) { if (retryCount >= Constants.RetryCount) { throw; } cancellationToken.WaitForInterval(TimeSpan.FromMilliseconds(pollingInterval)); retryCount++; } } return(jobDetailsResults); }
/// <summary> /// Method that waits for a jobDetails to complete. /// </summary> /// <param name="client">The Hadoop client to use.</param> /// <param name="jobId">The id of the job to wait for.</param> /// <param name="duration">The duration to wait before timing out.</param> /// <param name="cancellationToken"> /// The Cancellation Token for the request. /// </param> /// <returns>An awaitable task that represents the action.</returns> public static async Task <JobDetails> WaitForJobCompletionAsync( this IJobSubmissionClient client, string jobId, TimeSpan duration, CancellationToken cancellationToken) { client.ArgumentNotNull("client"); jobId.ArgumentNotNull("jobId"); JobDetails jobDetailsResults = new JobDetails() { JobId = jobId, StatusCode = JobStatusCode.Unknown }; var pollingInterval = GetPollingInterval(); var startTime = DateTime.UtcNow; var endTime = DateTime.UtcNow; while (jobDetailsResults.IsNotNull() && ((endTime = DateTime.UtcNow) - startTime) < duration && !(jobDetailsResults.StatusCode == JobStatusCode.Completed || jobDetailsResults.StatusCode == JobStatusCode.Failed || jobDetailsResults.StatusCode == JobStatusCode.Canceled)) { client.HandleClusterWaitNotifyEvent(jobDetailsResults); if (jobDetailsResults.StatusCode == JobStatusCode.Completed || jobDetailsResults.StatusCode == JobStatusCode.Failed) { break; } Thread.Sleep(pollingInterval); jobDetailsResults = await GetJobWithRetry(client, jobId, cancellationToken); } if (jobDetailsResults.StatusCode != JobStatusCode.Completed && jobDetailsResults.StatusCode != JobStatusCode.Failed && jobDetailsResults.StatusCode != JobStatusCode.Canceled && (endTime - startTime) >= duration) { throw new TimeoutException(string.Format(CultureInfo.InvariantCulture, "The requested task failed to complete in the allotted time ({0}).", duration)); } return(jobDetailsResults); }
internal IJobSubmissionClient GetClient(string cluster) { cluster.ArgumentNotNull("ClusterEndpoint"); IJobSubmissionClient client = null; ProfileClient profileClient = new ProfileClient(); string currentEnvironmentName = this.CurrentSubscription == null ? null : this.CurrentSubscription.Environment; var clientCredential = this.GetJobSubmissionClientCredentials( this.CurrentSubscription, profileClient.GetEnvironmentOrDefault(currentEnvironmentName), cluster, profileClient.Profile); if (clientCredential != null) { client = ServiceLocator.Instance.Locate <IAzureHDInsightJobSubmissionClientFactory>().Create(clientCredential); client.SetCancellationSource(this.tokenSource); if (this.Logger.IsNotNull()) { client.AddLogWriter(this.Logger); } return(client); } throw new InvalidOperationException("Expected either a Subscription or Credential parameter."); }
private static async Task <JobCreationResults> SubmitMapReduceJob( AzureHDInsightMapReduceJobDefinition azureMapReduceJobDefinition, IJobSubmissionClient client) { MapReduceJobCreateParameters mapReduceJobDefinition = azureMapReduceJobDefinition.ToMapReduceJobCreateParameters(); var jobCreationResults = await client.CreateMapReduceJobAsync(mapReduceJobDefinition); return(jobCreationResults); }
/// <summary> /// Method that waits for a jobDetails to complete. /// </summary> /// <param name="client">The Hadoop client to use.</param> /// <param name="job">The jobDetails to wait for.</param> /// <param name="duration">The duration to wait before timing out.</param> /// <param name="cancellationToken"> /// The Cancellation Token for the request. /// </param> /// <returns>An awaitable task that represents the action.</returns> public static async Task <JobDetails> WaitForJobCompletionAsync( this IJobSubmissionClient client, JobCreationResults job, TimeSpan duration, CancellationToken cancellationToken) { client.ArgumentNotNull("client"); job.ArgumentNotNull("jobDetails"); return(await client.WaitForJobCompletionAsync(job.JobId, duration, cancellationToken)); }
private static async Task <JobCreationResults> CreateSqoopJob( AzureHDInsightSqoopJobDefinition azureSqoopJobDefinition, IJobSubmissionClient client) { SqoopJobCreateParameters sqoopJobDefinition = azureSqoopJobDefinition.ToSqoopJobCreateParameters(); var jobCreationResults = await client.CreateSqoopJobAsync(sqoopJobDefinition); return(jobCreationResults); }
private static void WaitForJobCompletion(JobCreationResults jobResults, IJobSubmissionClient client) { JobDetails jobInProgress = client.GetJob(jobResults.JobId); while (jobInProgress.StatusCode != JobStatusCode.Completed && jobInProgress.StatusCode != JobStatusCode.Failed) { jobInProgress = client.GetJob(jobInProgress.JobId); Thread.Sleep(TimeSpan.FromSeconds(5)); } }
private static void WaitForJobCompletion(IJobSubmissionClient client, JobCreationResults jobResults) { JobDetails jobInProgress = client.GetJob(jobResults.JobId); while (jobInProgress.StatusCode != JobStatusCode.Completed && jobInProgress.StatusCode != JobStatusCode.Failed) { jobInProgress = client.GetJob(jobInProgress.JobId); Thread.Sleep(TimeSpan.FromSeconds(10)); } }
private static async Task <JobCreationResults> CreateSqoopJob( AzureHDInsightSqoopJobDefinition azureSqoopJobDefinition, IJobSubmissionClient client) { AssertQueryDoesNotContainRestrictedCharacters(azureSqoopJobDefinition.Command, "Command"); SqoopJobCreateParameters sqoopJobDefinition = azureSqoopJobDefinition.ToSqoopJobCreateParameters(); var jobCreationResults = await client.CreateSqoopJobAsync(sqoopJobDefinition); return(jobCreationResults); }
private static void WaitForJobCompletion(JobCreationResults jobResults, IJobSubmissionClient client) { // Wait for job completion, displaying progress JobDetails jobInProgress = client.GetJob(jobResults.JobId); Console.Write(jobResults.JobId); while (jobInProgress.StatusCode != JobStatusCode.Completed && jobInProgress.StatusCode != JobStatusCode.Failed) { jobInProgress = client.GetJob(jobInProgress.JobId); Thread.Sleep(TimeSpan.FromSeconds(5)); Console.Write("."); } Console.WriteLine(jobInProgress.StatusCode); }
public override async Task EndProcessing() { this.JobId.ArgumentNotNullOrEmpty("JobId"); IJobSubmissionClient client = this.GetClient(this.Cluster); if (client != null) { var jobDetail = await client.StopJobAsync(this.JobId); if (jobDetail != null) { this.Output.Add(new AzureHDInsightJob(jobDetail, this.Cluster)); } } }
//Helper Function to Wait while job executes private static void WaitForJobCompletion(JobCreationResults jobResults, IJobSubmissionClient client) { Trace.WriteLine("Entering WaitForJobCompletion method"); Trace.TraceInformation("Executing WaitForJobCompletion method " + DateTime.Now.ToLongTimeString()); JobDetails jobInProgress = client.GetJob(jobResults.JobId); while (jobInProgress.StatusCode != JobStatusCode.Completed && jobInProgress.StatusCode != JobStatusCode.Failed) { jobInProgress = client.GetJob(jobInProgress.JobId); Thread.Sleep(TimeSpan.FromSeconds(1)); Console.Write("."); } Trace.WriteLine("Leaving WaitForJobCompletion method"); }
/// <summary> /// Connects to HDInsight cluster. /// </summary> /// <param name="certificate">The certificate.</param> /// <param name="subscription">The subscription.</param> /// <param name="clusterName">Name of the cluster.</param> /// <param name="storageAccountName">Name of the storage account.</param> /// <param name="storageAccountKey">The storage account key.</param> public void Connect(string certificate, string subscription, string clusterName, string storageAccountName, string storageAccountKey) { // Obtain the certificate var store = new X509Store(); store.Open(OpenFlags.ReadOnly); var cert = store.Certificates.Cast <X509Certificate2>().FirstOrDefault(item => string.Compare(item.Thumbprint, certificate, CultureInfo.InvariantCulture, CompareOptions.IgnoreCase) == 0); if (cert == null) { AvroHdiSample.ReportError("Error: Counld not find the certificate on this machine!"); } // Connect to the cluster using the certificate and the subscription try { this.client = HDInsightClient.Connect(new HDInsightCertificateCredential(new Guid(subscription), cert)); } catch (Exception e) { AvroHdiSample.ReportError("Error while connecting to HDInsight service\n" + e); } this.cluster = this.client.GetCluster(clusterName); if (this.cluster == null) { AvroHdiSample.ReportError("Error while connecting to cluster: " + clusterName); } // Create a job client this.job = JobSubmissionClientFactory.Connect( new JobSubmissionCertificateCredential(new Guid(subscription), cert, clusterName)); // Create an Azure storage client // We will use this client to upload files to Azure storage account // which is used by HDInsight cluster. var storageAccount = CloudStorageAccount.Parse( "DefaultEndpointsProtocol=https;AccountName=" + storageAccountName + ";AccountKey=" + storageAccountKey); var blobClient = storageAccount.CreateCloudBlobClient(); this.container = blobClient.GetContainerReference(this.cluster.DefaultStorageAccount.Container); }
private static async Task <JobCreationResults> SubmitHiveJob( AzureHDInsightHiveJobDefinition azureHiveJobDefinition, IJobSubmissionClient client) { AssertQueryDoesNotContainRestrictedCharacters(azureHiveJobDefinition.Query, "Query"); var hiveJobDefinition = new HiveJobCreateParameters { JobName = azureHiveJobDefinition.JobName, Query = azureHiveJobDefinition.Query, File = azureHiveJobDefinition.File }; hiveJobDefinition.StatusFolder = azureHiveJobDefinition.StatusFolder; hiveJobDefinition.Arguments.AddRange(azureHiveJobDefinition.Arguments); hiveJobDefinition.Defines.AddRange(azureHiveJobDefinition.Defines); hiveJobDefinition.Files.AddRange(azureHiveJobDefinition.Files); var jobCreationResults = await client.CreateHiveJobAsync(hiveJobDefinition); return(jobCreationResults); }
private static async Task <JobCreationResults> SubmitHiveJob( AzureHDInsightHiveJobDefinition azureHiveJobDefinition, IJobSubmissionClient client) { var hiveJobDefinition = new HiveJobCreateParameters { JobName = azureHiveJobDefinition.JobName, Query = azureHiveJobDefinition.Query, File = azureHiveJobDefinition.File, RunAsFileJob = azureHiveJobDefinition.RunAsFileJob }; hiveJobDefinition.StatusFolder = azureHiveJobDefinition.StatusFolder; hiveJobDefinition.Arguments.AddRange(azureHiveJobDefinition.Arguments); hiveJobDefinition.Defines.AddRange(azureHiveJobDefinition.Defines); hiveJobDefinition.Files.AddRange(azureHiveJobDefinition.Files); var jobCreationResults = await client.CreateHiveJobAsync(hiveJobDefinition); return(jobCreationResults); }
public override void Initialize() { base.Initialize(); base.ApplyIndividualTestMockingOnly(); this.storageAccount = GetWellKnownStorageAccounts().First(); this.storageClient = new WabStorageAbstraction( new WindowsAzureStorageAccountCredentials { Key = this.storageAccount.Key, Name = this.storageAccount.Name, ContainerName = this.storageAccount.Container }); jobSubmissionClient = new HDInsightHadoopClient( new JobSubmissionCertificateCredential( Cluster.Value.SubscriptionId, GetValidCredentials().Certificate, Cluster.Value.Name)); this.CleanupCluster(); }
public override void Initialize() { base.Initialize(); base.ApplyIndividualTestMockingOnly(); this.storageAccount = GetWellKnownStorageAccounts().First(); this.storageClient = new WabStorageAbstraction( new WindowsAzureStorageAccountCredentials { Key = this.storageAccount.Key, Name = this.storageAccount.Name, ContainerName = this.storageAccount.Container }); jobSubmissionClient = new HDInsightHadoopClient( new JobSubmissionCertificateCredential( Cluster.Value.SubscriptionId, GetValidCredentials().Certificate, Cluster.Value.Name)); this.CleanupCluster(); }
public override async Task EndProcessing() { IJobSubmissionClient client = this.GetClient(this.Cluster); this.Output.Clear(); if (string.IsNullOrEmpty(this.JobId)) { var jobsList = await client.ListJobsAsync(); this.Output.AddRange(jobsList.Jobs.Select(hadoopJob => new AzureHDInsightJob(hadoopJob, this.Cluster))); } else { var jobDetail = await client.GetJobAsync(this.JobId); if (jobDetail != null) { this.Output.Add(new AzureHDInsightJob(jobDetail, this.Cluster)); } } }
private async Task GetJobOutput(string jobId) { this.JobId.ArgumentNotNullOrEmpty("jobId"); Stream outputStream = null; IJobSubmissionClient hadoopClient = this.GetClient(this.Cluster); switch (this.OutputType) { case JobOutputType.StandardError: outputStream = await hadoopClient.GetJobErrorLogsAsync(jobId); break; case JobOutputType.StandardOutput: outputStream = await hadoopClient.GetJobOutputAsync(jobId); break; case JobOutputType.TaskSummary: outputStream = await hadoopClient.GetJobTaskLogSummaryAsync(jobId); break; case JobOutputType.TaskLogs: this.TaskLogsDirectory.ArgumentNotNullOrEmpty("TaskLogsDirectory"); await hadoopClient.DownloadJobTaskLogsAsync(jobId, this.TaskLogsDirectory); var messageStream = new MemoryStream(); string downloadCompleteMessage = string.Format( CultureInfo.InvariantCulture, TaskLogDownloadCompleteTemplate, this.JobId, this.TaskLogsDirectory); byte[] messageBytes = Encoding.UTF8.GetBytes(downloadCompleteMessage); messageStream.Write(messageBytes, 0, messageBytes.Length); messageStream.Seek(0, SeekOrigin.Begin); outputStream = messageStream; break; } this.Output.Add(outputStream); }
private static void WaitForJobCompletion(JobCreationResults jobResults, IJobSubmissionClient client, String idUsuario, String subIdUsuario) { try { // Retrieve storage account from connection string. CloudStorageAccount storageAccount = CloudStorageAccount.Parse( CloudConfigurationManager.GetSetting("Microsoft.WindowsAzure.Plugins.Diagnostics.ConnectionString")); // Create the blob client. CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient(); // Retrieve reference to a previously created container. CloudBlobContainer container = blobClient.GetContainerReference(VariablesConfiguracion.containerName); CloudBlockBlob blockBlob = container.GetBlockBlobReference("pruebaProgreso-" + idUsuario + "." + subIdUsuario); JobDetails jobInProgress = client.GetJob(jobResults.JobId); while (jobInProgress.StatusCode != JobStatusCode.Completed && jobInProgress.StatusCode != JobStatusCode.Failed) { blockBlob.UploadText(VariablesConfiguracion.JOB_RUNNING); jobInProgress = client.GetJob(jobInProgress.JobId); } if (jobInProgress.StatusCode == JobStatusCode.Failed) { blockBlob.UploadText(VariablesConfiguracion.JOB_FAIL); } else if (jobInProgress.StatusCode == JobStatusCode.Completed) { blockBlob.UploadText(VariablesConfiguracion.JOB_SUCCESS); } } catch (Exception ex) { Trace.TraceError(ex.Message); throw; } finally { } }
/// <summary> /// Method that waits for a jobDetails to complete. /// </summary> /// <param name="client">The Hadoop client to use.</param> /// <param name="job">The jobDetails to wait for.</param> /// <param name="duration">The duration to wait before timing out.</param> /// <param name="cancellationToken"> /// The Cancellation Token for the request. /// </param> /// <returns>The jobDetails's pigJobCreateParameters.</returns> public static JobDetails WaitForJobCompletion( this IJobSubmissionClient client, JobCreationResults job, TimeSpan duration, CancellationToken cancellationToken) { return(WaitForJobCompletionAsync(client, job, duration, cancellationToken).WaitForResult()); }
private static void WaitForJobCompletion(JobCreationResults jobResults, IJobSubmissionClient client, String idUsuario, String subIdUsuario) { try { // Retrieve storage account from connection string. CloudStorageAccount storageAccount = CloudStorageAccount.Parse( CloudConfigurationManager.GetSetting("Microsoft.WindowsAzure.Plugins.Diagnostics.ConnectionString")); // Create the blob client. CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient(); // Retrieve reference to a previously created container. CloudBlobContainer container = blobClient.GetContainerReference(VariablesConfiguracion.containerName); CloudBlockBlob blockBlob = container.GetBlockBlobReference("pruebaProgreso-" + idUsuario + "." + subIdUsuario); JobDetails jobInProgress = client.GetJob(jobResults.JobId); while (jobInProgress.StatusCode != JobStatusCode.Completed && jobInProgress.StatusCode != JobStatusCode.Failed) { blockBlob.UploadText(VariablesConfiguracion.JOB_RUNNING); jobInProgress = client.GetJob(jobInProgress.JobId); } if (jobInProgress.StatusCode == JobStatusCode.Failed) { blockBlob.UploadText(VariablesConfiguracion.JOB_FAIL); } else if (jobInProgress.StatusCode == JobStatusCode.Completed) { blockBlob.UploadText(VariablesConfiguracion.JOB_SUCCESS); } } catch (Exception ex) { Trace.TraceError(ex.Message); throw; } finally { } }
private static async Task<JobDetails> GetJobWithRetry(IJobSubmissionClient client, string jobId, CancellationToken cancellationToken) { JobDetails jobDetailsResults = null; var pollingInterval = GetPollingInterval(); int retryCount = 0; while (jobDetailsResults.IsNull()) { try { jobDetailsResults = await client.GetJobAsync(jobId); break; } catch (HttpLayerException) { if (retryCount >= Constants.RetryCount) { throw; } cancellationToken.WaitForInterval(TimeSpan.FromMilliseconds(pollingInterval)); retryCount++; } } return jobDetailsResults; }
public override async Task EndProcessing() { this.JobDefinition.ArgumentNotNull("JobDefinition"); IJobSubmissionClient client = this.GetClient(this.Cluster); JobCreationResults jobCreationResults = null; if (this.JobDefinition.StatusFolder.IsNullOrEmpty()) { this.JobDefinition.StatusFolder = Guid.NewGuid().ToString(); } var azureMapReduceJobDefinition = this.JobDefinition as AzureHDInsightMapReduceJobDefinition; var azureHiveJobDefinition = this.JobDefinition as AzureHDInsightHiveJobDefinition; var azurePigJobDefinition = this.JobDefinition as AzureHDInsightPigJobDefinition; var azureStreamingJobDefinition = this.JobDefinition as AzureHDInsightStreamingMapReduceJobDefinition; var azureSqoopJobDefinition = this.JobDefinition as AzureHDInsightSqoopJobDefinition; if (azureMapReduceJobDefinition != null) { if (azureMapReduceJobDefinition.JobName.IsNullOrEmpty()) { azureMapReduceJobDefinition.JobName = azureMapReduceJobDefinition.ClassName; } jobCreationResults = await SubmitMapReduceJob(azureMapReduceJobDefinition, client); } else if (azureHiveJobDefinition != null) { if (azureHiveJobDefinition.JobName.IsNullOrEmpty()) { azureHiveJobDefinition.JobName = string.Format( CultureInfo.InvariantCulture, HiveJobNameFormat, GetJobNameFromQueryOrFile(azureHiveJobDefinition.Query, azureHiveJobDefinition.File)); } jobCreationResults = await SubmitHiveJob(azureHiveJobDefinition, client); } else if (azurePigJobDefinition != null) { jobCreationResults = await SubmitPigJob(azurePigJobDefinition, client); } else if (azureStreamingJobDefinition != null) { if (azureStreamingJobDefinition.JobName.IsNullOrEmpty()) { azureStreamingJobDefinition.JobName = GetLastSegment(azureStreamingJobDefinition.Mapper); } jobCreationResults = await CreateStreamingJob(azureStreamingJobDefinition, client); } else if (azureSqoopJobDefinition != null) { jobCreationResults = await CreateSqoopJob(azureSqoopJobDefinition, client); } else { throw new NotSupportedException( string.Format(CultureInfo.InvariantCulture, "Cannot start jobDetails of type : {0}.", this.JobDefinition.GetType())); } var startedJob = await client.GetJobAsync(jobCreationResults.JobId); if (startedJob.ErrorCode.IsNotNullOrEmpty()) { throw new InvalidOperationException("Failed to start jobDetails :" + startedJob.ErrorCode); } var jobDetail = new AzureHDInsightJob(startedJob, this.Cluster); this.Output.Add(jobDetail); }
private static async Task <JobCreationResults> CreateStreamingJob( AzureHDInsightStreamingMapReduceJobDefinition azureStreamingJobDefinition, IJobSubmissionClient client) { var streamingJobDefinition = new StreamingMapReduceJobCreateParameters { JobName = azureStreamingJobDefinition.JobName, Input = azureStreamingJobDefinition.Input, Output = azureStreamingJobDefinition.Output, Reducer = azureStreamingJobDefinition.Reducer, Combiner = azureStreamingJobDefinition.Combiner, Mapper = azureStreamingJobDefinition.Mapper }; streamingJobDefinition.StatusFolder = azureStreamingJobDefinition.StatusFolder; streamingJobDefinition.CommandEnvironment.AddRange(azureStreamingJobDefinition.CommandEnvironment); streamingJobDefinition.Arguments.AddRange(azureStreamingJobDefinition.Arguments); streamingJobDefinition.Defines.AddRange(azureStreamingJobDefinition.Defines); streamingJobDefinition.Files.AddRange(azureStreamingJobDefinition.Files); var jobCreationResults = await client.CreateStreamingJobAsync(streamingJobDefinition); return(jobCreationResults); }
private static async Task <JobCreationResults> SubmitPigJob(AzureHDInsightPigJobDefinition azurePigJobDefinition, IJobSubmissionClient client) { var pigJobDefinition = new PigJobCreateParameters { Query = azurePigJobDefinition.Query, File = azurePigJobDefinition.File }; pigJobDefinition.StatusFolder = azurePigJobDefinition.StatusFolder; pigJobDefinition.Arguments.AddRange(azurePigJobDefinition.Arguments); pigJobDefinition.Files.AddRange(azurePigJobDefinition.Files); var jobCreationResults = await client.CreatePigJobAsync(pigJobDefinition); return(jobCreationResults); }
/// <summary> /// Connects to HDInsight cluster. /// </summary> /// <param name="certificate">The certificate.</param> /// <param name="subscription">The subscription.</param> /// <param name="clusterName">Name of the cluster.</param> /// <param name="storageAccountName">Name of the storage account.</param> /// <param name="storageAccountKey">The storage account key.</param> public void Connect(string certificate, string subscription, string clusterName, string storageAccountName, string storageAccountKey) { // Obtain the certificate var store = new X509Store(); store.Open(OpenFlags.ReadOnly); var cert = store.Certificates.Cast<X509Certificate2>().FirstOrDefault(item => string.Compare(item.Thumbprint, certificate, CultureInfo.InvariantCulture, CompareOptions.IgnoreCase) == 0); if (cert == null) { AvroHdiSample.ReportError("Error: Counld not find the certificate on this machine!"); } // Connect to the cluster using the certificate and the subscription try { this.client = HDInsightClient.Connect(new HDInsightCertificateCredential(new Guid(subscription), cert)); } catch (Exception e) { AvroHdiSample.ReportError("Error while connecting to HDInsight service\n" + e); } this.cluster = this.client.GetCluster(clusterName); if (this.cluster == null) { AvroHdiSample.ReportError("Error while connecting to cluster: " + clusterName); } // Create a job client this.job = JobSubmissionClientFactory.Connect( new JobSubmissionCertificateCredential(new Guid(subscription), cert, clusterName)); // Create an Azure storage client // We will use this client to upload files to Azure storage account // which is used by HDInsight cluster. var storageAccount = CloudStorageAccount.Parse( "DefaultEndpointsProtocol=https;AccountName=" + storageAccountName + ";AccountKey=" + storageAccountKey); var blobClient = storageAccount.CreateCloudBlobClient(); this.container = blobClient.GetContainerReference(this.cluster.DefaultStorageAccount.Container); }
private static void WaitForJobCompletion(JobCreationResults jobResults, IJobSubmissionClient client) { // Wait for job completion, displaying progress JobDetails jobInProgress = client.GetJob(jobResults.JobId); Console.Write(jobResults.JobId); while (jobInProgress.StatusCode != JobStatusCode.Completed && jobInProgress.StatusCode != JobStatusCode.Failed) { jobInProgress = client.GetJob(jobInProgress.JobId); Thread.Sleep(TimeSpan.FromSeconds(5)); Console.Write("."); } Console.WriteLine(jobInProgress.StatusCode); }