Exemple #1
0
        /// <summary>
        /// Method that waits for a jobDetails to complete.
        /// </summary>
        /// <param name="client">The Hadoop client to use.</param>
        /// <param name="jobId">The id of the job to wait for.</param>
        /// <param name="duration">The duration to wait before timing out.</param>
        /// <param name="cancellationToken">
        /// The Cancellation Token for the request.
        /// </param>
        /// <returns>An awaitable task that represents the action.</returns>
        public static async Task <JobDetails> WaitForJobCompletionAsync(
            this IHadoopJobClient client, string jobId, TimeSpan duration, CancellationToken cancellationToken)
        {
            var jobDetailsResults = new JobDetails {
                JobId = jobId, StatusCode = JobStatusCode.Unknown
            };

            var startTime = DateTime.UtcNow;
            var endTime   = DateTime.UtcNow;

            while (ShouldRetryAgain(duration, jobDetailsResults, endTime, startTime))
            {
                if (JobIsFinished(jobDetailsResults))
                {
                    break;
                }

                await TaskEx.Delay(PollingInterval, cancellationToken);

                jobDetailsResults = await GetJobWithRetry(client, jobId, cancellationToken);

                endTime = DateTime.UtcNow;
            }

            if (JobIsNotFinished(jobDetailsResults) && WaitingTimeExceded(duration, endTime, startTime))
            {
                throw new TimeoutException("Timeout waiting for jobDetails completion");
            }

            return(jobDetailsResults);
        }
Exemple #2
0
        private static async Task <JobDetails> GetJobWithRetry(IHadoopJobClient client, string jobId,
                                                               CancellationToken cancellationToken)
        {
            JobDetails jobDetailsResults = null;

            int retryCount = 0;

            while (jobDetailsResults == null)
            {
                bool exceptionOccured;

                try
                {
                    jobDetailsResults = await client.GetJob(jobId);

                    break;
                }
                catch (Exception)
                {
                    if (retryCount >= RetryCount)
                    {
                        throw;
                    }

                    exceptionOccured = true;
                }

                if (exceptionOccured)
                {
                    await WaitForInterval(TimeSpan.FromMilliseconds(PollingInterval), cancellationToken);

                    retryCount++;
                }
            }

            return(jobDetailsResults);
        }
        private static async Task<JobDetails> GetJobWithRetry(IHadoopJobClient client, string jobId,
            CancellationToken cancellationToken)
        {
            JobDetails jobDetailsResults = null;

            int retryCount = 0;

            while (jobDetailsResults == null)
            {
                bool exceptionOccured;

                try
                {
                    jobDetailsResults = await client.GetJob(jobId);
                    break;
                }
                catch (Exception)
                {
                    if (retryCount >= RetryCount)
                        throw;

                    exceptionOccured = true;
                }

                if (exceptionOccured)
                {
                    await WaitForInterval(TimeSpan.FromMilliseconds(PollingInterval), cancellationToken);
                    retryCount++;
                }
            }

            return jobDetailsResults;
        }
Exemple #4
0
 public HiveClient(IHdfsClient hdfsClient, IHadoopJobClient jobClient, HiveClientConfig config)
 {
     _hdfsClient = hdfsClient;
     _jobClient  = jobClient;
     _config     = config;
 }
Exemple #5
0
 /// <summary>
 /// Method that waits for a jobDetails to complete.
 /// </summary>
 /// <param name="client">The Hadoop client to use.</param>
 /// <param name="job">The jobDetails to wait for.</param>
 /// <param name="duration">The duration to wait before timing out.</param>
 /// <param name="cancellationToken">
 /// The Cancellation Token for the request.
 /// </param>
 /// <returns>An awaitable task that represents the action.</returns>
 public static async Task <JobDetails> WaitForJobCompletionAsync(
     this IHadoopJobClient client, JobCreationResults job, TimeSpan duration, CancellationToken cancellationToken)
 {
     return(await client.WaitForJobCompletionAsync(job.JobId, duration, cancellationToken));
 }
Exemple #6
0
 /// <summary>
 /// Method that waits for a jobDetails to complete.
 /// </summary>
 /// <param name="client">The Hadoop client to use.</param>
 /// <param name="job">The jobDetails to wait for.</param>
 /// <param name="duration">The duration to wait before timing out.</param>
 /// <param name="cancellationToken">
 /// The Cancellation Token for the request.
 /// </param>
 /// <returns>The jobDetails's pigJobCreateParameters.</returns>
 public static JobDetails WaitForJobCompletion(
     this IHadoopJobClient client, JobCreationResults job, TimeSpan duration, CancellationToken cancellationToken)
 {
     return(WaitForJobCompletionAsync(client, job, duration, cancellationToken).Result);
 }