Ejemplo n.º 1
0
        Create(string streamName, KinesisFlowSettings settings = null, Func <IAmazonKinesis> client = null)
        {
            settings = settings ?? KinesisFlowSettings.Default;
            client   = client ?? DefaultClientFactory;

            return(Flow.Create <PutRecordsRequestEntry>()
                   .Throttle(settings.MaxRecordsPerSecond, Second, settings.MaxRecordsPerSecond, ThrottleMode.Shaping)
                   .Throttle(settings.MaxBytesPerSecond, Second, settings.MaxBytesPerSecond, GetPayloadByteSize, ThrottleMode.Shaping)
                   .Batch(settings.MaxBatchSize, ImmutableQueue.Create, (queue, request) => queue.Enqueue(request))
                   .Via(new KinesisFlowStage(streamName, settings.MaxRetries, settings.BackoffStrategy, settings.RetryInitialTimeout, client))
                   .SelectAsync(settings.Parallelism, task => task)
                   .SelectMany(result => result));
        }
Ejemplo n.º 2
0
 /// <summary>
 /// Transform this <see cref="Sink"/> by applying a function to each *incoming* upstream element before
 /// it is passed to the <see cref="Sink"/>
 ///
 /// Backpressures when original <see cref="Sink"/> backpressures
 ///
 /// Cancels when original <see cref="Sink"/> backpressures
 /// </summary>
 /// <typeparam name="TIn2">TBD</typeparam>
 /// <param name="function">TBD</param>
 /// <returns>TBD</returns>
 public Sink <TIn2, TMat> ContraMap <TIn2>(Func <TIn2, TIn> function)
 => Flow.FromFunction(function).ToMaterialized(this, Keep.Right);
Ejemplo n.º 3
0
 /// <summary>
 /// Protocol encoder that is used by <see cref="SimpleFramingProtocol"/>
 /// </summary>
 /// <param name="maximumMessageLength">TBD</param>
 /// <returns>TBD</returns>
 public static Flow <ByteString, ByteString, NotUsed> SimpleFramingProtocolEncoder(int maximumMessageLength)
 {
     return(Flow.Create <ByteString>().Via(new SimpleFramingProtocolEncoderStage(maximumMessageLength)));
 }
Ejemplo n.º 4
0
 /// <summary>
 /// Wrap the given <see cref="Flow"/> with a <see cref="Flow"/> that will restart it when it fails or complete using an exponential
 /// backoff.
 /// This <see cref="Flow"/> will not cancel, complete or emit a failure, until the opposite end of it has been cancelled or
 /// completed.Any termination by the <see cref="Flow"/> before that time will be handled by restarting it. Any termination
 /// signals sent to this <see cref="Flow"/> however will terminate the wrapped <see cref="Flow"/>, if it's running, and then the <see cref="Flow"/>
 /// will be allowed to terminate without being restarted.
 /// The restart process is inherently lossy, since there is no coordination between cancelling and the sending of
 /// messages. A termination signal from either end of the wrapped <see cref="Flow"/> will cause the other end to be terminated,
 /// and any in transit messages will be lost. During backoff, this <see cref="Flow"/> will backpressure.
 /// This uses the same exponential backoff algorithm as <see cref="Akka.Pattern.Backoff"/>.
 /// </summary>
 /// <param name="flowFactory">A factory for producing the <see cref="Flow"/>] to wrap.</param>
 /// <param name="minBackoff">Minimum (initial) duration until the child actor will started again, if it is terminated</param>
 /// <param name="maxBackoff">The exponential back-off is capped to this duration</param>
 /// <param name="randomFactor">After calculation of the exponential back-off an additional random delay based on this factor is added, e.g. `0.2` adds up to `20%` delay. In order to skip this additional delay pass in `0`.</param>
 public static Flow <TIn, TOut, NotUsed> WithBackoff <TIn, TOut, TMat>(Func <Flow <TIn, TOut, TMat> > flowFactory, TimeSpan minBackoff, TimeSpan maxBackoff, double randomFactor)
 => Flow.FromGraph(new RestartWithBackoffFlow <TIn, TOut, TMat>(flowFactory, minBackoff, maxBackoff, randomFactor));
Ejemplo n.º 5
0
 /// <summary>
 /// Wrap the given <see cref="Flow"/> with a <see cref="Flow"/> that will restart it when it fails using an exponential
 /// backoff. Notice that this <see cref="Flow"/> will not restart on completion of the wrapped flow.
 /// This <see cref="Flow"/> will not emit any failure
 /// The failures by the wrapped <see cref="Flow"/> will be handled by
 /// restarting the wrapping <see cref="Flow"/> as long as maxRestarts is not reached.
 /// Any termination signals sent to this <see cref="Flow"/> however will terminate the wrapped <see cref="Flow"/>, if it's
 /// running, and then the <see cref="Flow"/> will be allowed to terminate without being restarted.
 /// The restart process is inherently lossy, since there is no coordination between cancelling and the sending of
 /// messages. A termination signal from either end of the wrapped <see cref="Flow"/> will cause the other end to be terminated,
 /// nd any in transit messages will be lost. During backoff, this <see cref="Flow"/> will backpressure.
 /// This uses the same exponential backoff algorithm as <see cref="Akka.Pattern.Backoff"/>.
 /// </summary>
 /// <param name="flowFactory">A factory for producing the <see cref="Flow"/>] to wrap.</param>
 /// <param name="minBackoff">Minimum (initial) duration until the child actor will started again, if it is terminated</param>
 /// <param name="maxBackoff">The exponential back-off is capped to this duration</param>
 /// <param name="randomFactor">After calculation of the exponential back-off an additional random delay based on this factor is added, e.g. `0.2` adds up to `20%` delay. In order to skip this additional delay pass in `0`.</param>
 /// <param name="maxRestarts">The amount of restarts is capped to this amount within a time frame of minBackoff. Passing `0` will cause no restarts and a negative number will not cap the amount of restarts.</param>
 public static Flow <TIn, TOut, NotUsed> OnFailuresWithBackoff <TIn, TOut, TMat>(Func <Flow <TIn, TOut, TMat> > flowFactory, TimeSpan minBackoff, TimeSpan maxBackoff, double randomFactor, int maxRestarts)
 => Flow.FromGraph(new RestartWithBackoffFlow <TIn, TOut, TMat>(flowFactory, minBackoff, maxBackoff, randomFactor, true, maxRestarts));
Ejemplo n.º 6
0
 /// <summary>
 /// Create a <see cref="BidiFlow{TIn1,TOut1,TIn2,TOut2,TMat}"/> where the top and bottom flows are just one simple mapping
 /// stage each, expressed by the two functions.
 /// </summary>
 /// <typeparam name="TIn1">TBD</typeparam>
 /// <typeparam name="TOut1">TBD</typeparam>
 /// <typeparam name="TIn2">TBD</typeparam>
 /// <typeparam name="TOut2">TBD</typeparam>
 /// <param name="outbound">TBD</param>
 /// <param name="inbound">TBD</param>
 /// <returns>TBD</returns>
 public static BidiFlow <TIn1, TOut1, TIn2, TOut2, NotUsed> FromFunction <TIn1, TOut1, TIn2, TOut2>(Func <TIn1, TOut1> outbound, Func <TIn2, TOut2> inbound)
 {
     return(FromFlows(Flow.Create <TIn1>().Select(outbound), Flow.Create <TIn2>().Select(inbound)));
 }
Ejemplo n.º 7
0
 public static IGraph <FlowShape <T, IEnumerable <T> >, NotUsed> Create <T>(TimeSpan minInterval, int maxBatchSize)
 {
     return(Flow.Create <T>().GroupedWithin(maxBatchSize, minInterval).Via(new DelayFlow <IEnumerable <T> >(minInterval)));
 }