コード例 #1
0
        /// <summary>
        /// <see cref="ProducerThreadMethod"/> works in parallel with RPM's <see cref="RequestProcessingManager.WriteTo"/>.
        /// RPM supplies empty buffers to be filled with data into <see cref="RequestExecutionContext.BuffersRing"/> and consumes them on the other end.
        /// The data ring has very limited number of buffers.
        /// RPM is limited by network throughput and Producer's speed.
        /// Producer is limited by underlying storage driver, local processing speed and RPM's consumption of complete buffers.
        /// The difference between the two: RPM <see cref="RequestProcessingManager.WriteTo"/> is scheduled for execution by service infrastructure (WCF),
        /// whereas <see cref="DataEngine.ProducerThreadMethod"/> is scheduled by RPM itself, when it invokes <see cref="IDataEngine.BeginExecution"/>.
        /// </summary>
        void ProducerThreadMethod(RequestExecutionContext context)
        {
            PqlEngineSecurityContext.Set(new PqlClientSecurityContext(
                                             context.AuthContext.UserId, "dummy", context.AuthContext.TenantId, context.AuthContext.ContextId));

            var executionPending = true;
            IDriverDataEnumerator sourceEnumerator = null;

            try
            {
                // row number is needed for "rownum()" Pql function
                context.ClauseEvaluationContext.RowNumber = 0;

                // Our production is limited by the network throughput.
                // Production will also be aborted if the destination sink stops accepting.
                // In that case, ConsumingEnumerable will either throw or stop yielding.
                bool havePendingDriverRow = false;
                foreach (var buffer in context.BuffersRing.ConsumeProcessingTasks(context.CancellationTokenSource.Token)
                         )
                {
                    buffer.Cleanup();

                    try
                    {
                        if (executionPending)
                        {
                            executionPending = false;

                            // read network protocol message
                            // parse-compile expressions and collect information for execution plan
                            // generate response headers and fetch data from Redis
                            // this place fails most often, because of Pql compilation or storage driver connectivity failures
                            StartProduction(context, buffer, out sourceEnumerator);

                            if (context.Request.ReturnDataset)
                            {
                                // write response headers BEFORE the query processing is completed
                                // records affected and whatever other stats will be zero
                                Serializer.SerializeWithLengthPrefix(buffer.Stream, context.ResponseHeaders, PrefixStyle.Base128);
                            }
                        }

                        // go through retrieved data
                        havePendingDriverRow = ((DataEngine)context.Engine).WriteItems(
                            context, buffer, sourceEnumerator, havePendingDriverRow);

                        // some consistency checks
                        if (context.Request.ReturnDataset)
                        {
                            if (havePendingDriverRow && buffer.RowsOutput == 0)
                            {
                                throw new Exception("Internal error: should not have pending row when no data is produced");
                            }
                        }
                        else
                        {
                            if (havePendingDriverRow)
                            {
                                throw new Exception("Internal error: should not have pending row when no dataset is requested");
                            }

                            if (buffer.Stream.Length > 0)
                            {
                                throw new Exception("Internal error: should not have written anything to stream when no dataset is requested");
                            }
                        }

                        // time to quit?
                        // no dataset requested, don't have any more data, or enough rows accumulated for requested page of results?
                        if (buffer.RowsOutput == 0 || buffer.IsFailed || !context.Request.ReturnDataset)
                        {
                            if (!context.Request.ReturnDataset)
                            {
                                // if there is no dataset sent, write response headers AFTER processing the query
                                // records affected and whatever other stats are meaningful
                                context.ResponseHeaders.RecordsAffected = context.RecordsAffected;
                                Serializer.SerializeWithLengthPrefix(buffer.Stream, context.ResponseHeaders, PrefixStyle.Base128);
                            }

                            break;
                        }
                    }
                    catch (Exception e)
                    {
                        buffer.Error = e;
                        context.TrySetLastError(e);
                        m_tracer.Exception(e);

                        // this will go to client, and overwrite whatever we managed to put into buffer before failure
                        using (var writer = new PqlErrorDataWriter(1, e, false))
                        {
                            buffer.Stream.SetLength(0);
                            writer.WriteTo(buffer.Stream);
                        }

                        return;
                    }
                    finally
                    {
                        // return the buffer back to the ring in any case
                        context.BuffersRing.ReturnCompletedTask(buffer);
                    }
                }
            }
            catch (OperationCanceledException e)
            {
                context.Cancel(e);
            }
            catch (Exception e)
            {
                if (Environment.HasShutdownStarted)
                {
                    // nobody cares now
                    return;
                }

                var cts = context.CancellationTokenSource;
                if (cts != null && !cts.IsCancellationRequested)
                {
                    m_tracer.Exception(e);
                    context.Cancel(e);
                }
            }
            finally
            {
                var ring = context.BuffersRing;
                if (ring != null)
                {
                    ring.CompleteAddingCompletedTasks();
                }

                if (sourceEnumerator != null)
                {
                    // release driver-level resources & locks
                    sourceEnumerator.Dispose();
                }
            }
        }
コード例 #2
0
        /// <summary>
        /// RequestProcessingManager's (RPM) <see cref="WriteTo"/> works in parallel with <see cref="DataEngine.ProducerThreadMethod"/>.
        /// RPM supplies empty buffers to be filled with data into <see cref="RequestExecutionContext.BuffersRing"/> and consumes them on the other end.
        /// The data ring has very limited number of buffers.
        /// RPM is limited by network throughput and Producer's speed.
        /// Producer is limited by underlying storage driver, local processing speed and RPM's consumption of complete buffers.
        /// The difference between the two: RPM <see cref="WriteTo"/> is scheduled for execution by service infrastructure (WCF),
        /// whereas <see cref="DataEngine.ProducerThreadMethod"/> is scheduled by RPM itself, when it invokes <see cref="IDataEngine.BeginExecution"/>.
        /// </summary>
        public void WriteTo(Stream output)
        {
            // We will not report to client any unhandled errors from producer thread.
            // As of our local errors, we can only send them to client BEFORE the first block of data is streamed out.
            var canReportLocalErrors = true;

            try
            {
                using (var binaryWriter = new BinaryWriter(output, Encoding.UTF8, true))
                {
                    // rotate through the buffers, flush them to output until producer stops
                    var cancellation      = m_executionContext.CancellationTokenSource.Token;
                    var buffersRing       = m_executionContext.BuffersRing;
                    var lastCompletedTask = buffersRing.TakeCompletedTask(cancellation);
                    while (lastCompletedTask != null)
                    {
                        // no more reporting of local errors after first block has been generated
                        // note that this block may itself contain error information, but this is unrelated
                        canReportLocalErrors = false;

                        var stream  = lastCompletedTask.Stream;
                        var toWrite = checked ((int)stream.Length);
                        if (toWrite == 0)
                        {
                            break;
                        }

                        // write block
                        var buffer = stream.GetBuffer();
                        BufferedReaderStream.WriteBlock(binaryWriter, toWrite, buffer);

                        ReportStats(toWrite, lastCompletedTask.RowsOutput);

                        // this task might have produced error information instead of real data
                        // or it might have finished producing rows
                        // in both cases, break further processing
                        if (lastCompletedTask.IsFailed || lastCompletedTask.RowsOutput == 0)
                        {
                            break;
                        }

                        // return buffer to the ring
                        buffersRing.AddTaskForProcessing(lastCompletedTask, cancellation);
                        // take next one
                        lastCompletedTask = buffersRing.TakeCompletedTask(cancellation);
                    }

                    BufferedReaderStream.WriteStreamEndMarker(binaryWriter);

                    // no more data in the input sequence, or just have to stop producing
                    buffersRing.CompleteAddingTasksForProcessing();
                }
            }
            catch (Exception e)
            {
                m_executionContext.Cancel(e);

                if (canReportLocalErrors)
                {
                    using (var errorWriter = new PqlErrorDataWriter(1, m_executionContext.LastError, true))
                    {
                        errorWriter.WriteTo(output);
                    }
                }

                if (!(e is OperationCanceledException))
                {
                    throw;
                }
            }
        }