/// <summary>
/// Method that waits for a jobDetails to complete.
/// </summary>
/// <param name="client">The Hadoop client to use.</param>
/// <param name="jobId">The id of the job to wait for.</param>
/// <param name="duration">The duration to wait before timing out.</param>
/// <param name="cancellationToken">
/// The Cancellation Token for the request.
/// </param>
/// <returns>An awaitable task that represents the action.</returns>
public static async Task <JobDetails> WaitForJobCompletionAsync(
this IHadoopJobClient client, string jobId, TimeSpan duration, CancellationToken cancellationToken)
{
var jobDetailsResults = new JobDetails {
JobId = jobId, StatusCode = JobStatusCode.Unknown
};
var startTime = DateTime.UtcNow;
var endTime = DateTime.UtcNow;
while (ShouldRetryAgain(duration, jobDetailsResults, endTime, startTime))
{
if (JobIsFinished(jobDetailsResults))
{
break;
}
await TaskEx.Delay(PollingInterval, cancellationToken);
jobDetailsResults = await GetJobWithRetry(client, jobId, cancellationToken);
endTime = DateTime.UtcNow;
}
if (JobIsNotFinished(jobDetailsResults) && WaitingTimeExceded(duration, endTime, startTime))
{
throw new TimeoutException("Timeout waiting for jobDetails completion");
}
return(jobDetailsResults);
}
private static async Task <JobDetails> GetJobWithRetry(IHadoopJobClient client, string jobId,
CancellationToken cancellationToken)
{
JobDetails jobDetailsResults = null;
int retryCount = 0;
while (jobDetailsResults == null)
{
bool exceptionOccured;
try
{
jobDetailsResults = await client.GetJob(jobId);
break;
}
catch (Exception)
{
if (retryCount >= RetryCount)
{
throw;
}
exceptionOccured = true;
}
if (exceptionOccured)
{
await WaitForInterval(TimeSpan.FromMilliseconds(PollingInterval), cancellationToken);
retryCount++;
}
}
return(jobDetailsResults);
}
private static async Task<JobDetails> GetJobWithRetry(IHadoopJobClient client, string jobId,
CancellationToken cancellationToken)
{
JobDetails jobDetailsResults = null;
int retryCount = 0;
while (jobDetailsResults == null)
{
bool exceptionOccured;
try
{
jobDetailsResults = await client.GetJob(jobId);
break;
}
catch (Exception)
{
if (retryCount >= RetryCount)
throw;
exceptionOccured = true;
}
if (exceptionOccured)
{
await WaitForInterval(TimeSpan.FromMilliseconds(PollingInterval), cancellationToken);
retryCount++;
}
}
return jobDetailsResults;
}
public HiveClient(IHdfsClient hdfsClient, IHadoopJobClient jobClient, HiveClientConfig config)
{
_hdfsClient = hdfsClient;
_jobClient = jobClient;
_config = config;
}
/// <summary>
/// Method that waits for a jobDetails to complete.
/// </summary>
/// <param name="client">The Hadoop client to use.</param>
/// <param name="job">The jobDetails to wait for.</param>
/// <param name="duration">The duration to wait before timing out.</param>
/// <param name="cancellationToken">
/// The Cancellation Token for the request.
/// </param>
/// <returns>An awaitable task that represents the action.</returns>
public static async Task <JobDetails> WaitForJobCompletionAsync(
this IHadoopJobClient client, JobCreationResults job, TimeSpan duration, CancellationToken cancellationToken)
{
return(await client.WaitForJobCompletionAsync(job.JobId, duration, cancellationToken));
}
/// <summary>
/// Method that waits for a jobDetails to complete.
/// </summary>
/// <param name="client">The Hadoop client to use.</param>
/// <param name="job">The jobDetails to wait for.</param>
/// <param name="duration">The duration to wait before timing out.</param>
/// <param name="cancellationToken">
/// The Cancellation Token for the request.
/// </param>
/// <returns>The jobDetails's pigJobCreateParameters.</returns>
public static JobDetails WaitForJobCompletion(
this IHadoopJobClient client, JobCreationResults job, TimeSpan duration, CancellationToken cancellationToken)
{
return(WaitForJobCompletionAsync(client, job, duration, cancellationToken).Result);
}