private void ProcessInput(
            out List <OutOfOrderPartitionedStreamEvent <int, int> > diagnosticEvents,
            out List <PartitionedStreamEvent <int, int> > dataEvents)
        {
            var qc      = new QueryContainer();
            var ingress = qc.RegisterInput(this.input.ToObservable(), this.disorderPolicy,
                                           PartitionedFlushPolicy.None, PeriodicPunctuationPolicy.None(), this.lowWatermarkPolicy);

            var outOfOrderEvents = new List <OutOfOrderPartitionedStreamEvent <int, int> >();

            ingress.GetDroppedAdjustedEventsDiagnostic().Subscribe(o => outOfOrderEvents.Add(o));

            var output  = new List <PartitionedStreamEvent <int, int> >();
            var egress  = qc.RegisterOutput(ingress).ForEachAsync(o => output.Add(o));
            var process = qc.Restore();

            process.Flush();
            egress.Wait();

            diagnosticEvents = outOfOrderEvents;
            dataEvents       = output;
        }
Esempio n. 2
0
 public FlushOnBatchBoundary_OnCompletedEndOfStream() : base(DisorderPolicy.Throw(), FlushPolicy.FlushOnBatchBoundary, PeriodicPunctuationPolicy.None(), OnCompletedPolicy.EndOfStream)
 {
 }
Esempio n. 3
0
 public FlushNone_OnCompletedEndOfStream() : base(DisorderPolicy.Throw(), FlushPolicy.None, PeriodicPunctuationPolicy.None(), OnCompletedPolicy.EndOfStream)
 {
 }
Esempio n. 4
0
 public FlushOnPunctuation_OnCompletedFlush() : base(DisorderPolicy.Throw(), FlushPolicy.FlushOnPunctuation, PeriodicPunctuationPolicy.None(), OnCompletedPolicy.Flush)
 {
 }
        public void LowWatermarkGenerationWithReorderLatency()
        {
            SetupQuery(
                DisorderPolicy.Drop(reorderLatency: 50),
                PartitionedFlushPolicy.None,
                PeriodicPunctuationPolicy.None(),
                PeriodicLowWatermarkPolicy.Time(generationPeriod: 100, lowWatermarkTimestampLag: 10));

            // This establishes three partitions with keys 0,1,2
            this.input.OnNext(PartitionedStreamEvent.CreatePoint(0, 0, 0));
            this.input.OnNext(PartitionedStreamEvent.CreatePoint(1, 0, 1));
            this.input.OnNext(PartitionedStreamEvent.CreatePoint(2, 0, 2));

            // This will not yet generate a low watermark, since 99 - 0 is less than the generation period of 100
            this.input.OnNext(PartitionedStreamEvent.CreatePoint(0, 99, 0));

            // This will generate a low watermark, since (110 - lowWatermarkTimestampLag of 10) is greater than the
            // last watermark timestamp (in this case, none) + generation period (100).
            // The timestamp will be 110 - (lowWatermarkTimestampLag of 10) = 100.
            this.input.OnNext(PartitionedStreamEvent.CreatePoint(2, 110, 0));

            // These will be dropped due to our PartitionedDisorderPolicy.Drop, since the low watermark is now 100,
            // even as they approach the low watermark, since any event before the low watermark is out of order.
            this.input.OnNext(PartitionedStreamEvent.CreatePoint(0, 90, 0));
            this.input.OnNext(PartitionedStreamEvent.CreatePoint(1, 99, 1));

            // These will be honored, since they are >= low watermark
            this.input.OnNext(PartitionedStreamEvent.CreatePoint(0, 100, 1));
            this.input.OnNext(PartitionedStreamEvent.CreatePoint(1, 105, 2));

            // This will be reordered since it is >= low watermark and within the partition's
            // high watermark (110) - reorderLatency (50)
            this.input.OnNext(PartitionedStreamEvent.CreatePoint(2, 100, 0));

            this.input.OnNext(PartitionedStreamEvent.CreatePoint(0, 120, 0));
            this.input.OnNext(PartitionedStreamEvent.CreatePoint(1, 150, 1));
            this.input.OnNext(PartitionedStreamEvent.CreatePoint(2, 190, 2));

            // This will generate a low watermark, since (260 - lowWatermarkTimestampLag of 10) is greater than the
            // last watermark timestamp (100) + generation period (100).
            // The timestamp will be 260 - (lowWatermarkTimestampLag of 10) = 250.
            this.input.OnNext(PartitionedStreamEvent.CreatePoint(2, 260, 0));

            // An explicitly ingressed low watermark will update/reset the generationPeriod
            this.input.OnNext(PartitionedStreamEvent.CreateLowWatermark <int, int>(300));

            // Even though (370 - lowWatermarkTimestampLag of 10) is greater than the last automatically generated
            // low watermark timestamp (100) + the generation period (100)); the lowWatermarkGeneration period was
            // updated to start at 300, so the next generated low watermark will be at timestamp >=
            // (300 + generationPeriod of 100)
            this.input.OnNext(PartitionedStreamEvent.CreatePoint(0, 370, 1));

            // This will generate a low watermark, since (410 - lowWatermarkTimestampLag of 10) is >= the last low
            // watermark timestamp (300) + the generation period (100).
            // The timestamp will be 410 - (lowWatermarkTimestampLag of 10) = 400.
            this.input.OnNext(PartitionedStreamEvent.CreatePoint(0, 410, 1));

            var expected = new PartitionedStreamEvent <int, int>[]
            {
                PartitionedStreamEvent.CreatePoint(0, 0, 0),
                PartitionedStreamEvent.CreatePoint(1, 0, 1),
                PartitionedStreamEvent.CreatePoint(2, 0, 2),

                PartitionedStreamEvent.CreatePoint(0, 99, 0),

                // Generated automatically because of PeriodicLowWatermarkPolicy in response to point (key 2:100)
                PartitionedStreamEvent.CreateLowWatermark <int, int>(100),

                PartitionedStreamEvent.CreatePoint(0, 100, 1),
                PartitionedStreamEvent.CreatePoint(2, 100, 0),
                PartitionedStreamEvent.CreatePoint(1, 105, 2),
                PartitionedStreamEvent.CreatePoint(2, 110, 0),

                PartitionedStreamEvent.CreatePoint(0, 120, 0),
                PartitionedStreamEvent.CreatePoint(1, 150, 1),
                PartitionedStreamEvent.CreatePoint(2, 190, 2),

                // Generated automatically because of PeriodicLowWatermarkPolicy in response to point (key 2:260),
                // snapped to the leftmost generationPeriod boundary
                PartitionedStreamEvent.CreateLowWatermark <int, int>(200),
                PartitionedStreamEvent.CreatePoint(2, 260, 0),

                // Explicitly ingressed low watermark
                PartitionedStreamEvent.CreateLowWatermark <int, int>(300),

                PartitionedStreamEvent.CreatePoint(0, 370, 1),

                // Generated automatically because of PeriodicLowWatermarkPolicy in response to point (key 0:410)
                PartitionedStreamEvent.CreateLowWatermark <int, int>(400),
                PartitionedStreamEvent.CreatePoint(0, 410, 1),

                // Generated by the default OnCompletedPolicy EndOfStream
                PartitionedStreamEvent.CreateLowWatermark <int, int>(StreamEvent.InfinitySyncTime),
            };

            FinishQuery(expected);
        }