Create(string streamName, KinesisFlowSettings settings = null, Func <IAmazonKinesis> client = null)
{
settings = settings ?? KinesisFlowSettings.Default;
client = client ?? DefaultClientFactory;
return(Flow.Create <PutRecordsRequestEntry>()
.Throttle(settings.MaxRecordsPerSecond, Second, settings.MaxRecordsPerSecond, ThrottleMode.Shaping)
.Throttle(settings.MaxBytesPerSecond, Second, settings.MaxBytesPerSecond, GetPayloadByteSize, ThrottleMode.Shaping)
.Batch(settings.MaxBatchSize, ImmutableQueue.Create, (queue, request) => queue.Enqueue(request))
.Via(new KinesisFlowStage(streamName, settings.MaxRetries, settings.BackoffStrategy, settings.RetryInitialTimeout, client))
.SelectAsync(settings.Parallelism, task => task)
.SelectMany(result => result));
}
/// <summary>
/// Protocol encoder that is used by <see cref="SimpleFramingProtocol"/>
/// </summary>
/// <param name="maximumMessageLength">TBD</param>
/// <returns>TBD</returns>
public static Flow <ByteString, ByteString, NotUsed> SimpleFramingProtocolEncoder(int maximumMessageLength)
{
return(Flow.Create <ByteString>().Via(new SimpleFramingProtocolEncoderStage(maximumMessageLength)));
}
/// <summary>
/// Create a <see cref="BidiFlow{TIn1,TOut1,TIn2,TOut2,TMat}"/> where the top and bottom flows are just one simple mapping
/// stage each, expressed by the two functions.
/// </summary>
/// <typeparam name="TIn1">TBD</typeparam>
/// <typeparam name="TOut1">TBD</typeparam>
/// <typeparam name="TIn2">TBD</typeparam>
/// <typeparam name="TOut2">TBD</typeparam>
/// <param name="outbound">TBD</param>
/// <param name="inbound">TBD</param>
/// <returns>TBD</returns>
public static BidiFlow <TIn1, TOut1, TIn2, TOut2, NotUsed> FromFunction <TIn1, TOut1, TIn2, TOut2>(Func <TIn1, TOut1> outbound, Func <TIn2, TOut2> inbound)
{
return(FromFlows(Flow.Create <TIn1>().Select(outbound), Flow.Create <TIn2>().Select(inbound)));
}
public static IGraph <FlowShape <T, IEnumerable <T> >, NotUsed> Create <T>(TimeSpan minInterval, int maxBatchSize)
{
return(Flow.Create <T>().GroupedWithin(maxBatchSize, minInterval).Via(new DelayFlow <IEnumerable <T> >(minInterval)));
}
/// <summary>
/// Returns a Flow that implements a "brace counting" based framing stage for emitting valid JSON chunks.
/// It scans the incoming data stream for valid JSON objects and returns chunks of ByteStrings containing only those valid chunks.
///
/// Typical examples of data that one may want to frame using this stage include:
///
/// <para>
/// **Very large arrays**:
/// {{{
/// [{"id": 1}, {"id": 2}, [...], {"id": 999}]
/// }}}
/// </para>
///
/// <para>
/// **Multiple concatenated JSON objects** (with, or without commas between them):
/// {{{
/// {"id": 1}, {"id": 2}, [...], {"id": 999}
/// }}}
/// </para>
///
/// The framing works independently of formatting, i.e. it will still emit valid JSON elements even if two
/// elements are separated by multiple newlines or other whitespace characters. And of course is insensitive
/// (and does not impact the emitting frame) to the JSON object's internal formatting.
///
/// </summary>
/// <param name="maximumObjectLength">The maximum length of allowed frames while decoding. If the maximum length is exceeded this Flow will fail the stream.</param>
/// <returns>TBD</returns>
public static Flow <ByteString, ByteString, NotUsed> ObjectScanner(int maximumObjectLength)
{
return(Flow.Create <ByteString>().Via(new Scanner(maximumObjectLength)));
}
