Create(string streamName, KinesisFlowSettings settings = null, Func <IAmazonKinesis> client = null)
{
settings = settings ?? KinesisFlowSettings.Default;
client = client ?? DefaultClientFactory;
return(Flow.Create <PutRecordsRequestEntry>()
.Throttle(settings.MaxRecordsPerSecond, Second, settings.MaxRecordsPerSecond, ThrottleMode.Shaping)
.Throttle(settings.MaxBytesPerSecond, Second, settings.MaxBytesPerSecond, GetPayloadByteSize, ThrottleMode.Shaping)
.Batch(settings.MaxBatchSize, ImmutableQueue.Create, (queue, request) => queue.Enqueue(request))
.Via(new KinesisFlowStage(streamName, settings.MaxRetries, settings.BackoffStrategy, settings.RetryInitialTimeout, client))
.SelectAsync(settings.Parallelism, task => task)
.SelectMany(result => result));
}
/// <summary> /// Transform this <see cref="Sink"/> by applying a function to each *incoming* upstream element before /// it is passed to the <see cref="Sink"/> /// /// Backpressures when original <see cref="Sink"/> backpressures /// /// Cancels when original <see cref="Sink"/> backpressures /// </summary> /// <typeparam name="TIn2">TBD</typeparam> /// <param name="function">TBD</param> /// <returns>TBD</returns> public Sink <TIn2, TMat> ContraMap <TIn2>(Func <TIn2, TIn> function) => Flow.FromFunction(function).ToMaterialized(this, Keep.Right);
/// <summary>
/// Protocol encoder that is used by <see cref="SimpleFramingProtocol"/>
/// </summary>
/// <param name="maximumMessageLength">TBD</param>
/// <returns>TBD</returns>
public static Flow <ByteString, ByteString, NotUsed> SimpleFramingProtocolEncoder(int maximumMessageLength)
{
return(Flow.Create <ByteString>().Via(new SimpleFramingProtocolEncoderStage(maximumMessageLength)));
}
/// <summary> /// Wrap the given <see cref="Flow"/> with a <see cref="Flow"/> that will restart it when it fails or complete using an exponential /// backoff. /// This <see cref="Flow"/> will not cancel, complete or emit a failure, until the opposite end of it has been cancelled or /// completed.Any termination by the <see cref="Flow"/> before that time will be handled by restarting it. Any termination /// signals sent to this <see cref="Flow"/> however will terminate the wrapped <see cref="Flow"/>, if it's running, and then the <see cref="Flow"/> /// will be allowed to terminate without being restarted. /// The restart process is inherently lossy, since there is no coordination between cancelling and the sending of /// messages. A termination signal from either end of the wrapped <see cref="Flow"/> will cause the other end to be terminated, /// and any in transit messages will be lost. During backoff, this <see cref="Flow"/> will backpressure. /// This uses the same exponential backoff algorithm as <see cref="Akka.Pattern.Backoff"/>. /// </summary> /// <param name="flowFactory">A factory for producing the <see cref="Flow"/>] to wrap.</param> /// <param name="minBackoff">Minimum (initial) duration until the child actor will started again, if it is terminated</param> /// <param name="maxBackoff">The exponential back-off is capped to this duration</param> /// <param name="randomFactor">After calculation of the exponential back-off an additional random delay based on this factor is added, e.g. `0.2` adds up to `20%` delay. In order to skip this additional delay pass in `0`.</param> public static Flow <TIn, TOut, NotUsed> WithBackoff <TIn, TOut, TMat>(Func <Flow <TIn, TOut, TMat> > flowFactory, TimeSpan minBackoff, TimeSpan maxBackoff, double randomFactor) => Flow.FromGraph(new RestartWithBackoffFlow <TIn, TOut, TMat>(flowFactory, minBackoff, maxBackoff, randomFactor));
/// <summary> /// Wrap the given <see cref="Flow"/> with a <see cref="Flow"/> that will restart it when it fails using an exponential /// backoff. Notice that this <see cref="Flow"/> will not restart on completion of the wrapped flow. /// This <see cref="Flow"/> will not emit any failure /// The failures by the wrapped <see cref="Flow"/> will be handled by /// restarting the wrapping <see cref="Flow"/> as long as maxRestarts is not reached. /// Any termination signals sent to this <see cref="Flow"/> however will terminate the wrapped <see cref="Flow"/>, if it's /// running, and then the <see cref="Flow"/> will be allowed to terminate without being restarted. /// The restart process is inherently lossy, since there is no coordination between cancelling and the sending of /// messages. A termination signal from either end of the wrapped <see cref="Flow"/> will cause the other end to be terminated, /// nd any in transit messages will be lost. During backoff, this <see cref="Flow"/> will backpressure. /// This uses the same exponential backoff algorithm as <see cref="Akka.Pattern.Backoff"/>. /// </summary> /// <param name="flowFactory">A factory for producing the <see cref="Flow"/>] to wrap.</param> /// <param name="minBackoff">Minimum (initial) duration until the child actor will started again, if it is terminated</param> /// <param name="maxBackoff">The exponential back-off is capped to this duration</param> /// <param name="randomFactor">After calculation of the exponential back-off an additional random delay based on this factor is added, e.g. `0.2` adds up to `20%` delay. In order to skip this additional delay pass in `0`.</param> /// <param name="maxRestarts">The amount of restarts is capped to this amount within a time frame of minBackoff. Passing `0` will cause no restarts and a negative number will not cap the amount of restarts.</param> public static Flow <TIn, TOut, NotUsed> OnFailuresWithBackoff <TIn, TOut, TMat>(Func <Flow <TIn, TOut, TMat> > flowFactory, TimeSpan minBackoff, TimeSpan maxBackoff, double randomFactor, int maxRestarts) => Flow.FromGraph(new RestartWithBackoffFlow <TIn, TOut, TMat>(flowFactory, minBackoff, maxBackoff, randomFactor, true, maxRestarts));
/// <summary>
/// Create a <see cref="BidiFlow{TIn1,TOut1,TIn2,TOut2,TMat}"/> where the top and bottom flows are just one simple mapping
/// stage each, expressed by the two functions.
/// </summary>
/// <typeparam name="TIn1">TBD</typeparam>
/// <typeparam name="TOut1">TBD</typeparam>
/// <typeparam name="TIn2">TBD</typeparam>
/// <typeparam name="TOut2">TBD</typeparam>
/// <param name="outbound">TBD</param>
/// <param name="inbound">TBD</param>
/// <returns>TBD</returns>
public static BidiFlow <TIn1, TOut1, TIn2, TOut2, NotUsed> FromFunction <TIn1, TOut1, TIn2, TOut2>(Func <TIn1, TOut1> outbound, Func <TIn2, TOut2> inbound)
{
return(FromFlows(Flow.Create <TIn1>().Select(outbound), Flow.Create <TIn2>().Select(inbound)));
}
public static IGraph <FlowShape <T, IEnumerable <T> >, NotUsed> Create <T>(TimeSpan minInterval, int maxBatchSize)
{
return(Flow.Create <T>().GroupedWithin(maxBatchSize, minInterval).Via(new DelayFlow <IEnumerable <T> >(minInterval)));
}