Create(string streamName, KinesisFlowSettings settings = null, Func <IAmazonKinesis> client = null) { settings = settings ?? KinesisFlowSettings.Default; client = client ?? DefaultClientFactory; return(Flow.Create <PutRecordsRequestEntry>() .Throttle(settings.MaxRecordsPerSecond, Second, settings.MaxRecordsPerSecond, ThrottleMode.Shaping) .Throttle(settings.MaxBytesPerSecond, Second, settings.MaxBytesPerSecond, GetPayloadByteSize, ThrottleMode.Shaping) .Batch(settings.MaxBatchSize, ImmutableQueue.Create, (queue, request) => queue.Enqueue(request)) .Via(new KinesisFlowStage(streamName, settings.MaxRetries, settings.BackoffStrategy, settings.RetryInitialTimeout, client)) .SelectAsync(settings.Parallelism, task => task) .SelectMany(result => result)); }
/// <summary> /// Protocol encoder that is used by <see cref="SimpleFramingProtocol"/> /// </summary> /// <param name="maximumMessageLength">TBD</param> /// <returns>TBD</returns> public static Flow <ByteString, ByteString, NotUsed> SimpleFramingProtocolEncoder(int maximumMessageLength) { return(Flow.Create <ByteString>().Via(new SimpleFramingProtocolEncoderStage(maximumMessageLength))); }
/// <summary> /// Create a <see cref="BidiFlow{TIn1,TOut1,TIn2,TOut2,TMat}"/> where the top and bottom flows are just one simple mapping /// stage each, expressed by the two functions. /// </summary> /// <typeparam name="TIn1">TBD</typeparam> /// <typeparam name="TOut1">TBD</typeparam> /// <typeparam name="TIn2">TBD</typeparam> /// <typeparam name="TOut2">TBD</typeparam> /// <param name="outbound">TBD</param> /// <param name="inbound">TBD</param> /// <returns>TBD</returns> public static BidiFlow <TIn1, TOut1, TIn2, TOut2, NotUsed> FromFunction <TIn1, TOut1, TIn2, TOut2>(Func <TIn1, TOut1> outbound, Func <TIn2, TOut2> inbound) { return(FromFlows(Flow.Create <TIn1>().Select(outbound), Flow.Create <TIn2>().Select(inbound))); }
public static IGraph <FlowShape <T, IEnumerable <T> >, NotUsed> Create <T>(TimeSpan minInterval, int maxBatchSize) { return(Flow.Create <T>().GroupedWithin(maxBatchSize, minInterval).Via(new DelayFlow <IEnumerable <T> >(minInterval))); }
/// <summary> /// Returns a Flow that implements a "brace counting" based framing stage for emitting valid JSON chunks. /// It scans the incoming data stream for valid JSON objects and returns chunks of ByteStrings containing only those valid chunks. /// /// Typical examples of data that one may want to frame using this stage include: /// /// <para> /// **Very large arrays**: /// {{{ /// [{"id": 1}, {"id": 2}, [...], {"id": 999}] /// }}} /// </para> /// /// <para> /// **Multiple concatenated JSON objects** (with, or without commas between them): /// {{{ /// {"id": 1}, {"id": 2}, [...], {"id": 999} /// }}} /// </para> /// /// The framing works independently of formatting, i.e. it will still emit valid JSON elements even if two /// elements are separated by multiple newlines or other whitespace characters. And of course is insensitive /// (and does not impact the emitting frame) to the JSON object's internal formatting. /// /// </summary> /// <param name="maximumObjectLength">The maximum length of allowed frames while decoding. If the maximum length is exceeded this Flow will fail the stream.</param> /// <returns>TBD</returns> public static Flow <ByteString, ByteString, NotUsed> ObjectScanner(int maximumObjectLength) { return(Flow.Create <ByteString>().Via(new Scanner(maximumObjectLength))); }