Example #1
0
        public Task <int> PostResultAsync(IQueuedTaskToken token, int timeoutMilliseconds)
        {
            CheckNotDisposedOrThrow();
            CheckRunningOrThrow();

            if (token == null)
            {
                throw new ArgumentNullException(nameof(token));
            }

            long requestId = Interlocked.Increment(ref mLastRequestId);

            PostgreSqlTaskResultQueueProcessRequest processRequest =
                new PostgreSqlTaskResultQueueProcessRequest(requestId,
                                                            token.LastQueuedTaskResult,
                                                            timeoutMilliseconds: timeoutMilliseconds,
                                                            maxFailCount: 3);

            mResultProcessingQueue.Add(processRequest);
            IncrementPostResultCount();

            return(processRequest.Task.WithCleanup((prev) =>
            {
                if (processRequest.IsTimedOut)
                {
                    IncrementResultWriteRequestTimeoutCount();
                }
                processRequest.Dispose();
            }));
        }
Example #2
0
        private async Task RunProcessingLoopAsync()
        {
            CancellationToken stopToken = mResultProcessingStopRequest
                                          .Token;

            if (stopToken.IsCancellationRequested)
            {
                return;
            }

            while (true)
            {
                //We need to use a queue here - as we process the batch,
                //	we consume each element and, in case of an error
                //	that affects all of them,
                //	we would fail only the remaining ones, not the ones
                //	that have been successfully processed
                Queue <PostgreSqlTaskResultQueueProcessRequest> currentBatch =
                    new Queue <PostgreSqlTaskResultQueueProcessRequest>();

                try
                {
                    stopToken.ThrowIfCancellationRequested();

                    //Try to dequeue and block if no item is available
                    PostgreSqlTaskResultQueueProcessRequest processItem =
                        mResultProcessingQueue.Take(stopToken);

                    currentBatch.Enqueue(processItem);

                    //See if there are other items available
                    //	and add them to current batch
                    while (currentBatch.Count < RESULT_QUEUE_PROCESSING_BATCH_SIZE && mResultProcessingQueue.TryTake(out processItem))
                    {
                        currentBatch.Enqueue(processItem);
                    }

                    //Process the entire batch - don't observe
                    //	cancellation token
                    await ProcessResultBatchAsync(currentBatch);
                }
                catch (OperationCanceledException)
                {
                    mLogger.Debug("Cancellation requested. Breaking result processing loop...");

                    //Best effort to cancel all tasks
                    foreach (PostgreSqlTaskResultQueueProcessRequest rq in mResultProcessingQueue.ToArray())
                    {
                        rq.SetCancelled();
                    }

                    break;
                }
                catch (Exception exc)
                {
                    //Add them back to processing queue to be retried
                    foreach (PostgreSqlTaskResultQueueProcessRequest rq in currentBatch)
                    {
                        rq.SetFailed(exc);
                        if (rq.CanBeRetried)
                        {
                            mResultProcessingQueue.Add(rq);
                        }
                    }

                    currentBatch.Clear();
                    mLogger.Error("Error processing results", exc);
                }
                finally
                {
                    //Clear batch and start over
                    currentBatch.Clear();
                }
            }
        }
Example #3
0
        private async Task ProcessResultBatchAsync(Queue <PostgreSqlTaskResultQueueProcessRequest> currentBatch)
        {
            MonotonicTimestamp startWrite = MonotonicTimestamp
                                            .Now();

            //An explicit choice has been made not to use transactions
            //	since failing to update a result MUST NOT
            //	cause the other successful updates to be rolled back.
            using (NpgsqlConnection conn = await OpenConnectionAsync(CancellationToken.None))
                using (NpgsqlCommand updateCmd = new NpgsqlCommand(mUpdateSql, conn))
                {
                    NpgsqlParameter pStatus = updateCmd.Parameters
                                              .Add("t_status", NpgsqlDbType.Integer);
                    NpgsqlParameter pLastError = updateCmd.Parameters
                                                 .Add("t_last_error", NpgsqlDbType.Text);
                    NpgsqlParameter pErrorCount = updateCmd.Parameters
                                                  .Add("t_error_count", NpgsqlDbType.Integer);
                    NpgsqlParameter pLastErrorIsRecoverable = updateCmd.Parameters
                                                              .Add("t_last_error_recoverable", NpgsqlDbType.Boolean);
                    NpgsqlParameter pProcessingTime = updateCmd.Parameters
                                                      .Add("t_processing_time_milliseconds", NpgsqlDbType.Bigint);
                    NpgsqlParameter pFinalizedAt = updateCmd.Parameters
                                                   .Add("t_processing_finalized_at_ts", NpgsqlDbType.TimestampTz);
                    NpgsqlParameter pId = updateCmd.Parameters
                                          .Add("t_id", NpgsqlDbType.Uuid);

                    await updateCmd.PrepareAsync();

                    while (currentBatch.Count > 0)
                    {
                        PostgreSqlTaskResultQueueProcessRequest processRq =
                            currentBatch.Dequeue();

                        try
                        {
                            pStatus.Value = ( int )processRq.ResultToUpdate.Status;

                            string strLastError = processRq.ResultToUpdate.LastError.ToJson();
                            if (strLastError != null)
                            {
                                pLastError.Value = strLastError;
                            }
                            else
                            {
                                pLastError.Value = DBNull.Value;
                            }

                            pErrorCount.Value             = processRq.ResultToUpdate.ErrorCount;
                            pLastErrorIsRecoverable.Value = processRq.ResultToUpdate.LastErrorIsRecoverable;
                            pProcessingTime.Value         = processRq.ResultToUpdate.ProcessingTimeMilliseconds;

                            if (processRq.ResultToUpdate.ProcessingFinalizedAtTs.HasValue)
                            {
                                pFinalizedAt.Value = processRq.ResultToUpdate.ProcessingFinalizedAtTs;
                            }
                            else
                            {
                                pFinalizedAt.Value = DBNull.Value;
                            }

                            pId.Value = processRq.ResultToUpdate.Id;

                            int affectedRows = await updateCmd.ExecuteNonQueryAsync();

                            processRq.SetCompleted(affectedRows);

                            IncrementResultWriteCount(MonotonicTimestamp
                                                      .Since(startWrite));
                        }
                        catch (OperationCanceledException)
                        {
                            processRq.SetCancelled();
                            throw;
                        }
                        catch (Exception exc)
                        {
                            processRq.SetFailed(exc);
                            if (processRq.CanBeRetried)
                            {
                                mResultProcessingQueue.Add(processRq);
                            }

                            mLogger.Error("Error processing result", exc);
                        }
                    }

                    await conn.CloseAsync();
                }
        }