public async Task ProcessRecord()
{
if (string.IsNullOrWhiteSpace(this.JobId) || string.IsNullOrWhiteSpace(this.Cluster))
{
this.JobId = this.Job.JobId;
this.Cluster = this.Job.Cluster;
}
IJobSubmissionClient client = this.GetClient(this.Cluster);
client.JobStatusEvent += this.ClientOnJobStatus;
JobDetails jobDetail = null;
if (ReduceWaitTime)
{
jobDetail = await client.GetJobAsync(this.JobId);
while (jobDetail.StatusCode != JobStatusCode.Completed && jobDetail.StatusCode != JobStatusCode.Failed &&
jobDetail.StatusCode != JobStatusCode.Canceled)
{
jobDetail = await client.GetJobAsync(this.JobId);
}
}
else
{
var job = new JobCreationResults {
JobId = this.JobId
};
jobDetail = await client.WaitForJobCompletionAsync(job, TimeSpan.FromSeconds(this.WaitTimeoutInSeconds), this.tokenSource.Token);
}
this.Output.Add(new AzureHDInsightJob(jobDetail, this.Cluster));
}
private static async Task <JobDetails> GetJobWithRetry(IJobSubmissionClient client, string jobId, CancellationToken cancellationToken)
{
JobDetails jobDetailsResults = null;
var pollingInterval = GetPollingInterval();
int retryCount = 0;
while (jobDetailsResults.IsNull())
{
try
{
jobDetailsResults = await client.GetJobAsync(jobId);
break;
}
catch (HttpLayerException)
{
if (retryCount >= Constants.RetryCount)
{
throw;
}
cancellationToken.WaitForInterval(TimeSpan.FromMilliseconds(pollingInterval));
retryCount++;
}
}
return(jobDetailsResults);
}
/// <summary>
/// Method that waits for a jobDetails to complete.
/// </summary>
/// <param name="client">The Hadoop client to use.</param>
/// <param name="jobId">The id of the job to wait for.</param>
/// <param name="duration">The duration to wait before timing out.</param>
/// <param name="cancellationToken">
/// The Cancellation Token for the request.
/// </param>
/// <returns>An awaitable task that represents the action.</returns>
public static async Task <JobDetails> WaitForJobCompletionAsync(
this IJobSubmissionClient client, string jobId, TimeSpan duration, CancellationToken cancellationToken)
{
client.ArgumentNotNull("client");
jobId.ArgumentNotNull("jobId");
JobDetails jobDetailsResults = new JobDetails()
{
JobId = jobId, StatusCode = JobStatusCode.Unknown
};
var pollingInterval = GetPollingInterval();
var startTime = DateTime.UtcNow;
var endTime = DateTime.UtcNow;
while (jobDetailsResults.IsNotNull() && ((endTime = DateTime.UtcNow) - startTime) < duration &&
!(jobDetailsResults.StatusCode == JobStatusCode.Completed || jobDetailsResults.StatusCode == JobStatusCode.Failed ||
jobDetailsResults.StatusCode == JobStatusCode.Canceled))
{
client.HandleClusterWaitNotifyEvent(jobDetailsResults);
if (jobDetailsResults.StatusCode == JobStatusCode.Completed || jobDetailsResults.StatusCode == JobStatusCode.Failed)
{
break;
}
Thread.Sleep(pollingInterval);
jobDetailsResults = await GetJobWithRetry(client, jobId, cancellationToken);
}
if (jobDetailsResults.StatusCode != JobStatusCode.Completed && jobDetailsResults.StatusCode != JobStatusCode.Failed &&
jobDetailsResults.StatusCode != JobStatusCode.Canceled && (endTime - startTime) >= duration)
{
throw new TimeoutException(string.Format(CultureInfo.InvariantCulture, "The requested task failed to complete in the allotted time ({0}).", duration));
}
return(jobDetailsResults);
}
internal IJobSubmissionClient GetClient(string cluster)
{
cluster.ArgumentNotNull("ClusterEndpoint");
IJobSubmissionClient client = null;
ProfileClient profileClient = new ProfileClient();
string currentEnvironmentName = this.CurrentSubscription == null ? null : this.CurrentSubscription.Environment;
var clientCredential = this.GetJobSubmissionClientCredentials(
this.CurrentSubscription,
profileClient.GetEnvironmentOrDefault(currentEnvironmentName),
cluster,
profileClient.Profile);
if (clientCredential != null)
{
client = ServiceLocator.Instance.Locate <IAzureHDInsightJobSubmissionClientFactory>().Create(clientCredential);
client.SetCancellationSource(this.tokenSource);
if (this.Logger.IsNotNull())
{
client.AddLogWriter(this.Logger);
}
return(client);
}
throw new InvalidOperationException("Expected either a Subscription or Credential parameter.");
}
private static async Task <JobCreationResults> SubmitMapReduceJob(
AzureHDInsightMapReduceJobDefinition azureMapReduceJobDefinition, IJobSubmissionClient client)
{
MapReduceJobCreateParameters mapReduceJobDefinition = azureMapReduceJobDefinition.ToMapReduceJobCreateParameters();
var jobCreationResults = await client.CreateMapReduceJobAsync(mapReduceJobDefinition);
return(jobCreationResults);
}
/// <summary>
/// Method that waits for a jobDetails to complete.
/// </summary>
/// <param name="client">The Hadoop client to use.</param>
/// <param name="job">The jobDetails to wait for.</param>
/// <param name="duration">The duration to wait before timing out.</param>
/// <param name="cancellationToken">
/// The Cancellation Token for the request.
/// </param>
/// <returns>An awaitable task that represents the action.</returns>
public static async Task <JobDetails> WaitForJobCompletionAsync(
this IJobSubmissionClient client, JobCreationResults job, TimeSpan duration, CancellationToken cancellationToken)
{
client.ArgumentNotNull("client");
job.ArgumentNotNull("jobDetails");
return(await client.WaitForJobCompletionAsync(job.JobId, duration, cancellationToken));
}
private static async Task <JobCreationResults> CreateSqoopJob(
AzureHDInsightSqoopJobDefinition azureSqoopJobDefinition, IJobSubmissionClient client)
{
SqoopJobCreateParameters sqoopJobDefinition = azureSqoopJobDefinition.ToSqoopJobCreateParameters();
var jobCreationResults = await client.CreateSqoopJobAsync(sqoopJobDefinition);
return(jobCreationResults);
}
private static void WaitForJobCompletion(JobCreationResults jobResults, IJobSubmissionClient client)
{
JobDetails jobInProgress = client.GetJob(jobResults.JobId);
while (jobInProgress.StatusCode != JobStatusCode.Completed && jobInProgress.StatusCode != JobStatusCode.Failed)
{
jobInProgress = client.GetJob(jobInProgress.JobId);
Thread.Sleep(TimeSpan.FromSeconds(5));
}
}
private static void WaitForJobCompletion(IJobSubmissionClient client, JobCreationResults jobResults)
{
JobDetails jobInProgress = client.GetJob(jobResults.JobId);
while (jobInProgress.StatusCode != JobStatusCode.Completed && jobInProgress.StatusCode != JobStatusCode.Failed)
{
jobInProgress = client.GetJob(jobInProgress.JobId);
Thread.Sleep(TimeSpan.FromSeconds(10));
}
}
private static async Task <JobCreationResults> CreateSqoopJob(
AzureHDInsightSqoopJobDefinition azureSqoopJobDefinition, IJobSubmissionClient client)
{
AssertQueryDoesNotContainRestrictedCharacters(azureSqoopJobDefinition.Command, "Command");
SqoopJobCreateParameters sqoopJobDefinition = azureSqoopJobDefinition.ToSqoopJobCreateParameters();
var jobCreationResults = await client.CreateSqoopJobAsync(sqoopJobDefinition);
return(jobCreationResults);
}
private static void WaitForJobCompletion(JobCreationResults jobResults, IJobSubmissionClient client)
{
// Wait for job completion, displaying progress
JobDetails jobInProgress = client.GetJob(jobResults.JobId);
Console.Write(jobResults.JobId);
while (jobInProgress.StatusCode != JobStatusCode.Completed && jobInProgress.StatusCode != JobStatusCode.Failed)
{
jobInProgress = client.GetJob(jobInProgress.JobId);
Thread.Sleep(TimeSpan.FromSeconds(5));
Console.Write(".");
}
Console.WriteLine(jobInProgress.StatusCode);
}
public override async Task EndProcessing()
{
this.JobId.ArgumentNotNullOrEmpty("JobId");
IJobSubmissionClient client = this.GetClient(this.Cluster);
if (client != null)
{
var jobDetail = await client.StopJobAsync(this.JobId);
if (jobDetail != null)
{
this.Output.Add(new AzureHDInsightJob(jobDetail, this.Cluster));
}
}
}
//Helper Function to Wait while job executes
private static void WaitForJobCompletion(JobCreationResults jobResults, IJobSubmissionClient client)
{
Trace.WriteLine("Entering WaitForJobCompletion method");
Trace.TraceInformation("Executing WaitForJobCompletion method " + DateTime.Now.ToLongTimeString());
JobDetails jobInProgress = client.GetJob(jobResults.JobId);
while (jobInProgress.StatusCode != JobStatusCode.Completed &&
jobInProgress.StatusCode != JobStatusCode.Failed)
{
jobInProgress = client.GetJob(jobInProgress.JobId);
Thread.Sleep(TimeSpan.FromSeconds(1));
Console.Write(".");
}
Trace.WriteLine("Leaving WaitForJobCompletion method");
}
/// <summary>
/// Connects to HDInsight cluster.
/// </summary>
/// <param name="certificate">The certificate.</param>
/// <param name="subscription">The subscription.</param>
/// <param name="clusterName">Name of the cluster.</param>
/// <param name="storageAccountName">Name of the storage account.</param>
/// <param name="storageAccountKey">The storage account key.</param>
public void Connect(string certificate, string subscription, string clusterName, string storageAccountName, string storageAccountKey)
{
// Obtain the certificate
var store = new X509Store();
store.Open(OpenFlags.ReadOnly);
var cert = store.Certificates.Cast <X509Certificate2>().FirstOrDefault(item => string.Compare(item.Thumbprint, certificate, CultureInfo.InvariantCulture, CompareOptions.IgnoreCase) == 0);
if (cert == null)
{
AvroHdiSample.ReportError("Error: Counld not find the certificate on this machine!");
}
// Connect to the cluster using the certificate and the subscription
try
{
this.client = HDInsightClient.Connect(new HDInsightCertificateCredential(new Guid(subscription), cert));
}
catch (Exception e)
{
AvroHdiSample.ReportError("Error while connecting to HDInsight service\n" + e);
}
this.cluster = this.client.GetCluster(clusterName);
if (this.cluster == null)
{
AvroHdiSample.ReportError("Error while connecting to cluster: " + clusterName);
}
// Create a job client
this.job = JobSubmissionClientFactory.Connect(
new JobSubmissionCertificateCredential(new Guid(subscription), cert, clusterName));
// Create an Azure storage client
// We will use this client to upload files to Azure storage account
// which is used by HDInsight cluster.
var storageAccount = CloudStorageAccount.Parse(
"DefaultEndpointsProtocol=https;AccountName=" + storageAccountName + ";AccountKey=" + storageAccountKey);
var blobClient = storageAccount.CreateCloudBlobClient();
this.container = blobClient.GetContainerReference(this.cluster.DefaultStorageAccount.Container);
}
private static async Task <JobCreationResults> SubmitHiveJob(
AzureHDInsightHiveJobDefinition azureHiveJobDefinition, IJobSubmissionClient client)
{
AssertQueryDoesNotContainRestrictedCharacters(azureHiveJobDefinition.Query, "Query");
var hiveJobDefinition = new HiveJobCreateParameters
{
JobName = azureHiveJobDefinition.JobName,
Query = azureHiveJobDefinition.Query,
File = azureHiveJobDefinition.File
};
hiveJobDefinition.StatusFolder = azureHiveJobDefinition.StatusFolder;
hiveJobDefinition.Arguments.AddRange(azureHiveJobDefinition.Arguments);
hiveJobDefinition.Defines.AddRange(azureHiveJobDefinition.Defines);
hiveJobDefinition.Files.AddRange(azureHiveJobDefinition.Files);
var jobCreationResults = await client.CreateHiveJobAsync(hiveJobDefinition);
return(jobCreationResults);
}
private static async Task <JobCreationResults> SubmitHiveJob(
AzureHDInsightHiveJobDefinition azureHiveJobDefinition, IJobSubmissionClient client)
{
var hiveJobDefinition = new HiveJobCreateParameters
{
JobName = azureHiveJobDefinition.JobName,
Query = azureHiveJobDefinition.Query,
File = azureHiveJobDefinition.File,
RunAsFileJob = azureHiveJobDefinition.RunAsFileJob
};
hiveJobDefinition.StatusFolder = azureHiveJobDefinition.StatusFolder;
hiveJobDefinition.Arguments.AddRange(azureHiveJobDefinition.Arguments);
hiveJobDefinition.Defines.AddRange(azureHiveJobDefinition.Defines);
hiveJobDefinition.Files.AddRange(azureHiveJobDefinition.Files);
var jobCreationResults = await client.CreateHiveJobAsync(hiveJobDefinition);
return(jobCreationResults);
}
public override void Initialize()
{
base.Initialize();
base.ApplyIndividualTestMockingOnly();
this.storageAccount = GetWellKnownStorageAccounts().First();
this.storageClient =
new WabStorageAbstraction(
new WindowsAzureStorageAccountCredentials
{
Key = this.storageAccount.Key,
Name = this.storageAccount.Name,
ContainerName = this.storageAccount.Container
});
jobSubmissionClient =
new HDInsightHadoopClient(
new JobSubmissionCertificateCredential(
Cluster.Value.SubscriptionId,
GetValidCredentials().Certificate,
Cluster.Value.Name));
this.CleanupCluster();
}
public override void Initialize()
{
base.Initialize();
base.ApplyIndividualTestMockingOnly();
this.storageAccount = GetWellKnownStorageAccounts().First();
this.storageClient =
new WabStorageAbstraction(
new WindowsAzureStorageAccountCredentials
{
Key = this.storageAccount.Key,
Name = this.storageAccount.Name,
ContainerName = this.storageAccount.Container
});
jobSubmissionClient =
new HDInsightHadoopClient(
new JobSubmissionCertificateCredential(
Cluster.Value.SubscriptionId,
GetValidCredentials().Certificate,
Cluster.Value.Name));
this.CleanupCluster();
}
public override async Task EndProcessing()
{
IJobSubmissionClient client = this.GetClient(this.Cluster);
this.Output.Clear();
if (string.IsNullOrEmpty(this.JobId))
{
var jobsList = await client.ListJobsAsync();
this.Output.AddRange(jobsList.Jobs.Select(hadoopJob => new AzureHDInsightJob(hadoopJob, this.Cluster)));
}
else
{
var jobDetail = await client.GetJobAsync(this.JobId);
if (jobDetail != null)
{
this.Output.Add(new AzureHDInsightJob(jobDetail, this.Cluster));
}
}
}
private async Task GetJobOutput(string jobId)
{
this.JobId.ArgumentNotNullOrEmpty("jobId");
Stream outputStream = null;
IJobSubmissionClient hadoopClient = this.GetClient(this.Cluster);
switch (this.OutputType)
{
case JobOutputType.StandardError:
outputStream = await hadoopClient.GetJobErrorLogsAsync(jobId);
break;
case JobOutputType.StandardOutput:
outputStream = await hadoopClient.GetJobOutputAsync(jobId);
break;
case JobOutputType.TaskSummary:
outputStream = await hadoopClient.GetJobTaskLogSummaryAsync(jobId);
break;
case JobOutputType.TaskLogs:
this.TaskLogsDirectory.ArgumentNotNullOrEmpty("TaskLogsDirectory");
await hadoopClient.DownloadJobTaskLogsAsync(jobId, this.TaskLogsDirectory);
var messageStream = new MemoryStream();
string downloadCompleteMessage = string.Format(
CultureInfo.InvariantCulture, TaskLogDownloadCompleteTemplate, this.JobId, this.TaskLogsDirectory);
byte[] messageBytes = Encoding.UTF8.GetBytes(downloadCompleteMessage);
messageStream.Write(messageBytes, 0, messageBytes.Length);
messageStream.Seek(0, SeekOrigin.Begin);
outputStream = messageStream;
break;
}
this.Output.Add(outputStream);
}
private static void WaitForJobCompletion(JobCreationResults jobResults, IJobSubmissionClient client, String idUsuario, String subIdUsuario)
{
try
{
// Retrieve storage account from connection string.
CloudStorageAccount storageAccount = CloudStorageAccount.Parse(
CloudConfigurationManager.GetSetting("Microsoft.WindowsAzure.Plugins.Diagnostics.ConnectionString"));
// Create the blob client.
CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient();
// Retrieve reference to a previously created container.
CloudBlobContainer container = blobClient.GetContainerReference(VariablesConfiguracion.containerName);
CloudBlockBlob blockBlob = container.GetBlockBlobReference("pruebaProgreso-" + idUsuario + "." + subIdUsuario);
JobDetails jobInProgress = client.GetJob(jobResults.JobId);
while (jobInProgress.StatusCode != JobStatusCode.Completed && jobInProgress.StatusCode != JobStatusCode.Failed)
{
blockBlob.UploadText(VariablesConfiguracion.JOB_RUNNING);
jobInProgress = client.GetJob(jobInProgress.JobId);
}
if (jobInProgress.StatusCode == JobStatusCode.Failed)
{
blockBlob.UploadText(VariablesConfiguracion.JOB_FAIL);
}
else if (jobInProgress.StatusCode == JobStatusCode.Completed)
{
blockBlob.UploadText(VariablesConfiguracion.JOB_SUCCESS);
}
}
catch (Exception ex)
{
Trace.TraceError(ex.Message);
throw;
}
finally {
}
}
/// <summary>
/// Method that waits for a jobDetails to complete.
/// </summary>
/// <param name="client">The Hadoop client to use.</param>
/// <param name="job">The jobDetails to wait for.</param>
/// <param name="duration">The duration to wait before timing out.</param>
/// <param name="cancellationToken">
/// The Cancellation Token for the request.
/// </param>
/// <returns>The jobDetails's pigJobCreateParameters.</returns>
public static JobDetails WaitForJobCompletion(
this IJobSubmissionClient client, JobCreationResults job, TimeSpan duration, CancellationToken cancellationToken)
{
return(WaitForJobCompletionAsync(client, job, duration, cancellationToken).WaitForResult());
}
private static void WaitForJobCompletion(JobCreationResults jobResults, IJobSubmissionClient client, String idUsuario, String subIdUsuario)
{
try
{
// Retrieve storage account from connection string.
CloudStorageAccount storageAccount = CloudStorageAccount.Parse(
CloudConfigurationManager.GetSetting("Microsoft.WindowsAzure.Plugins.Diagnostics.ConnectionString"));
// Create the blob client.
CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient();
// Retrieve reference to a previously created container.
CloudBlobContainer container = blobClient.GetContainerReference(VariablesConfiguracion.containerName);
CloudBlockBlob blockBlob = container.GetBlockBlobReference("pruebaProgreso-" + idUsuario + "." + subIdUsuario);
JobDetails jobInProgress = client.GetJob(jobResults.JobId);
while (jobInProgress.StatusCode != JobStatusCode.Completed && jobInProgress.StatusCode != JobStatusCode.Failed)
{
blockBlob.UploadText(VariablesConfiguracion.JOB_RUNNING);
jobInProgress = client.GetJob(jobInProgress.JobId);
}
if (jobInProgress.StatusCode == JobStatusCode.Failed)
{
blockBlob.UploadText(VariablesConfiguracion.JOB_FAIL);
}
else if (jobInProgress.StatusCode == JobStatusCode.Completed)
{
blockBlob.UploadText(VariablesConfiguracion.JOB_SUCCESS);
}
}
catch (Exception ex)
{
Trace.TraceError(ex.Message);
throw;
}
finally {
}
}
private static async Task<JobDetails> GetJobWithRetry(IJobSubmissionClient client, string jobId, CancellationToken cancellationToken)
{
JobDetails jobDetailsResults = null;
var pollingInterval = GetPollingInterval();
int retryCount = 0;
while (jobDetailsResults.IsNull())
{
try
{
jobDetailsResults = await client.GetJobAsync(jobId);
break;
}
catch (HttpLayerException)
{
if (retryCount >= Constants.RetryCount)
{
throw;
}
cancellationToken.WaitForInterval(TimeSpan.FromMilliseconds(pollingInterval));
retryCount++;
}
}
return jobDetailsResults;
}
public override async Task EndProcessing()
{
this.JobDefinition.ArgumentNotNull("JobDefinition");
IJobSubmissionClient client = this.GetClient(this.Cluster);
JobCreationResults jobCreationResults = null;
if (this.JobDefinition.StatusFolder.IsNullOrEmpty())
{
this.JobDefinition.StatusFolder = Guid.NewGuid().ToString();
}
var azureMapReduceJobDefinition = this.JobDefinition as AzureHDInsightMapReduceJobDefinition;
var azureHiveJobDefinition = this.JobDefinition as AzureHDInsightHiveJobDefinition;
var azurePigJobDefinition = this.JobDefinition as AzureHDInsightPigJobDefinition;
var azureStreamingJobDefinition = this.JobDefinition as AzureHDInsightStreamingMapReduceJobDefinition;
var azureSqoopJobDefinition = this.JobDefinition as AzureHDInsightSqoopJobDefinition;
if (azureMapReduceJobDefinition != null)
{
if (azureMapReduceJobDefinition.JobName.IsNullOrEmpty())
{
azureMapReduceJobDefinition.JobName = azureMapReduceJobDefinition.ClassName;
}
jobCreationResults = await SubmitMapReduceJob(azureMapReduceJobDefinition, client);
}
else if (azureHiveJobDefinition != null)
{
if (azureHiveJobDefinition.JobName.IsNullOrEmpty())
{
azureHiveJobDefinition.JobName = string.Format(
CultureInfo.InvariantCulture,
HiveJobNameFormat,
GetJobNameFromQueryOrFile(azureHiveJobDefinition.Query, azureHiveJobDefinition.File));
}
jobCreationResults = await SubmitHiveJob(azureHiveJobDefinition, client);
}
else if (azurePigJobDefinition != null)
{
jobCreationResults = await SubmitPigJob(azurePigJobDefinition, client);
}
else if (azureStreamingJobDefinition != null)
{
if (azureStreamingJobDefinition.JobName.IsNullOrEmpty())
{
azureStreamingJobDefinition.JobName = GetLastSegment(azureStreamingJobDefinition.Mapper);
}
jobCreationResults = await CreateStreamingJob(azureStreamingJobDefinition, client);
}
else if (azureSqoopJobDefinition != null)
{
jobCreationResults = await CreateSqoopJob(azureSqoopJobDefinition, client);
}
else
{
throw new NotSupportedException(
string.Format(CultureInfo.InvariantCulture, "Cannot start jobDetails of type : {0}.", this.JobDefinition.GetType()));
}
var startedJob = await client.GetJobAsync(jobCreationResults.JobId);
if (startedJob.ErrorCode.IsNotNullOrEmpty())
{
throw new InvalidOperationException("Failed to start jobDetails :" + startedJob.ErrorCode);
}
var jobDetail = new AzureHDInsightJob(startedJob, this.Cluster);
this.Output.Add(jobDetail);
}
private static async Task <JobCreationResults> CreateStreamingJob(
AzureHDInsightStreamingMapReduceJobDefinition azureStreamingJobDefinition, IJobSubmissionClient client)
{
var streamingJobDefinition = new StreamingMapReduceJobCreateParameters
{
JobName = azureStreamingJobDefinition.JobName,
Input = azureStreamingJobDefinition.Input,
Output = azureStreamingJobDefinition.Output,
Reducer = azureStreamingJobDefinition.Reducer,
Combiner = azureStreamingJobDefinition.Combiner,
Mapper = azureStreamingJobDefinition.Mapper
};
streamingJobDefinition.StatusFolder = azureStreamingJobDefinition.StatusFolder;
streamingJobDefinition.CommandEnvironment.AddRange(azureStreamingJobDefinition.CommandEnvironment);
streamingJobDefinition.Arguments.AddRange(azureStreamingJobDefinition.Arguments);
streamingJobDefinition.Defines.AddRange(azureStreamingJobDefinition.Defines);
streamingJobDefinition.Files.AddRange(azureStreamingJobDefinition.Files);
var jobCreationResults = await client.CreateStreamingJobAsync(streamingJobDefinition);
return(jobCreationResults);
}
private static async Task <JobCreationResults> SubmitPigJob(AzureHDInsightPigJobDefinition azurePigJobDefinition, IJobSubmissionClient client)
{
var pigJobDefinition = new PigJobCreateParameters {
Query = azurePigJobDefinition.Query, File = azurePigJobDefinition.File
};
pigJobDefinition.StatusFolder = azurePigJobDefinition.StatusFolder;
pigJobDefinition.Arguments.AddRange(azurePigJobDefinition.Arguments);
pigJobDefinition.Files.AddRange(azurePigJobDefinition.Files);
var jobCreationResults = await client.CreatePigJobAsync(pigJobDefinition);
return(jobCreationResults);
}
/// <summary>
/// Connects to HDInsight cluster.
/// </summary>
/// <param name="certificate">The certificate.</param>
/// <param name="subscription">The subscription.</param>
/// <param name="clusterName">Name of the cluster.</param>
/// <param name="storageAccountName">Name of the storage account.</param>
/// <param name="storageAccountKey">The storage account key.</param>
public void Connect(string certificate, string subscription, string clusterName, string storageAccountName, string storageAccountKey)
{
// Obtain the certificate
var store = new X509Store();
store.Open(OpenFlags.ReadOnly);
var cert = store.Certificates.Cast<X509Certificate2>().FirstOrDefault(item => string.Compare(item.Thumbprint, certificate, CultureInfo.InvariantCulture, CompareOptions.IgnoreCase) == 0);
if (cert == null)
{
AvroHdiSample.ReportError("Error: Counld not find the certificate on this machine!");
}
// Connect to the cluster using the certificate and the subscription
try
{
this.client = HDInsightClient.Connect(new HDInsightCertificateCredential(new Guid(subscription), cert));
}
catch (Exception e)
{
AvroHdiSample.ReportError("Error while connecting to HDInsight service\n" + e);
}
this.cluster = this.client.GetCluster(clusterName);
if (this.cluster == null)
{
AvroHdiSample.ReportError("Error while connecting to cluster: " + clusterName);
}
// Create a job client
this.job = JobSubmissionClientFactory.Connect(
new JobSubmissionCertificateCredential(new Guid(subscription), cert, clusterName));
// Create an Azure storage client
// We will use this client to upload files to Azure storage account
// which is used by HDInsight cluster.
var storageAccount = CloudStorageAccount.Parse(
"DefaultEndpointsProtocol=https;AccountName=" + storageAccountName + ";AccountKey=" + storageAccountKey);
var blobClient = storageAccount.CreateCloudBlobClient();
this.container = blobClient.GetContainerReference(this.cluster.DefaultStorageAccount.Container);
}
private static void WaitForJobCompletion(JobCreationResults jobResults, IJobSubmissionClient client)
{
// Wait for job completion, displaying progress
JobDetails jobInProgress = client.GetJob(jobResults.JobId);
Console.Write(jobResults.JobId);
while (jobInProgress.StatusCode != JobStatusCode.Completed && jobInProgress.StatusCode != JobStatusCode.Failed)
{
jobInProgress = client.GetJob(jobInProgress.JobId);
Thread.Sleep(TimeSpan.FromSeconds(5));
Console.Write(".");
}
Console.WriteLine(jobInProgress.StatusCode);
}
