コード例 #1
0
        protected internal WriteApi(InfluxDBClientOptions options, WriteService service, WriteOptions writeOptions,
                                    InfluxDBClient influxDbClient)
        {
            Arguments.CheckNotNull(service, nameof(service));
            Arguments.CheckNotNull(writeOptions, nameof(writeOptions));
            Arguments.CheckNotNull(influxDbClient, nameof(_influxDbClient));

            _options        = options;
            _influxDbClient = influxDbClient;

            // backpreasure - is not implemented in C#
            //
            // => use unbound buffer
            //
            // https://github.com/dotnet/reactive/issues/19

            var observable = _subject.ObserveOn(writeOptions.WriteScheduler);

            var boundary = observable
                           .Buffer(TimeSpan.FromMilliseconds(writeOptions.FlushInterval), writeOptions.BatchSize,
                                   writeOptions.WriteScheduler)
                           .Merge(_flush);

            observable
            //
            // Batching
            //
            .Window(boundary)
            //
            // Group by key - same bucket, same org
            //
            .SelectMany(it => it.GroupBy(batchWrite => batchWrite.Options))
            //
            // Create Write Point = bucket, org, ... + data
            //
            .Select(grouped =>
            {
                var aggregate = grouped
                                .Aggregate(new StringBuilder(""), (builder, batchWrite) =>
                {
                    var data = batchWrite.ToLineProtocol();

                    if (string.IsNullOrEmpty(data))
                    {
                        return(builder);
                    }

                    if (builder.Length > 0)
                    {
                        builder.Append("\n");
                    }

                    return(builder.Append(data));
                }).Select(builder => builder.ToString());

                return(aggregate.Select(records => new BatchWriteRecord(grouped.Key, records)));
            })
            //
            // Jitter
            //
            .Select(source =>
            {
                if (writeOptions.JitterInterval <= 0)
                {
                    return(source);
                }

                return(source.Delay(_ => Observable.Timer(TimeSpan.FromMilliseconds(JitterDelay(writeOptions)), Scheduler.CurrentThread)));
            })
            .Concat()
            //
            // Map to Async request
            //
            .Where(batchWriteItem => !string.IsNullOrEmpty(batchWriteItem.ToLineProtocol()))
            .Select(batchWriteItem =>
            {
                var org          = batchWriteItem.Options.OrganizationId;
                var bucket       = batchWriteItem.Options.Bucket;
                var lineProtocol = batchWriteItem.ToLineProtocol();
                var precision    = batchWriteItem.Options.Precision;

                return(Observable
                       .Defer(() =>
                              service.PostWriteAsyncWithIRestResponse(org, bucket,
                                                                      Encoding.UTF8.GetBytes(lineProtocol), null,
                                                                      "identity", "text/plain; charset=utf-8", null, "application/json", null, precision)
                              .ToObservable())
                       .RetryWhen(f => f.SelectMany(e =>
                {
                    if (e is HttpException httpException)
                    {
                        //
                        // This types is not able to retry
                        //
                        if (httpException.Status != 429 && httpException.Status != 503)
                        {
                            throw httpException;
                        }

                        var retryInterval = (httpException.RetryAfter * 1000 ?? writeOptions.RetryInterval) +
                                            JitterDelay(writeOptions);

                        var retryable = new WriteRetriableErrorEvent(org, bucket, precision, lineProtocol,
                                                                     httpException, retryInterval);
                        Publish(retryable);

                        return Observable.Timer(TimeSpan.FromMilliseconds(retryInterval));
                    }

                    throw e;
                }))
                       .Select(result =>
                {
                    // ReSharper disable once ConvertIfStatementToReturnStatement
                    if (result.IsSuccessful)
                    {
                        return Notification.CreateOnNext(result);
                    }

                    return Notification.CreateOnError <IRestResponse>(HttpException.Create(result));
                })
                       .Catch <Notification <IRestResponse>, Exception>(ex =>
                {
                    var error = new WriteErrorEvent(org, bucket, precision, lineProtocol, ex);
                    Publish(error);

                    return Observable.Return(Notification.CreateOnError <IRestResponse>(ex));
                }).Do(res =>
                {
                    if (res.Kind == NotificationKind.OnNext)
                    {
                        var success = new WriteSuccessEvent(org, bucket, precision, lineProtocol);
                        Publish(success);
                    }
                }));
            })
            .Concat()
            .Subscribe(
                notification =>
            {
                switch (notification.Kind)
                {
                case NotificationKind.OnNext:
                    Trace.WriteLine($"The batch item: {notification} was processed successfully.");
                    break;

                case NotificationKind.OnError:
                    Trace.WriteLine(
                        $"The batch item wasn't processed successfully because: {notification.Exception}");
                    break;

                default:
                    Trace.WriteLine($"The batch item: {notification} was processed");
                    break;
                }
            },
                exception => Trace.WriteLine($"The unhandled exception occurs: {exception}"),
                () => Trace.WriteLine("The WriteApi was disposed."));
        }
コード例 #2
0
        protected internal WriteApi(
            InfluxDBClientOptions options,
            WriteService service,
            WriteOptions writeOptions,
            IDomainObjectMapper mapper,
            InfluxDBClient influxDbClient,
            IObservable <Unit> disposeCommand)
        {
            Arguments.CheckNotNull(service, nameof(service));
            Arguments.CheckNotNull(writeOptions, nameof(writeOptions));
            Arguments.CheckNotNull(mapper, nameof(mapper));
            Arguments.CheckNotNull(influxDbClient, nameof(_influxDbClient));
            Arguments.CheckNotNull(disposeCommand, nameof(disposeCommand));

            _options        = options;
            _mapper         = mapper;
            _influxDbClient = influxDbClient;

            _unsubscribeDisposeCommand = disposeCommand.Subscribe(_ => Dispose());

            // backpreasure - is not implemented in C#
            //
            // => use unbound buffer
            //
            // https://github.com/dotnet/reactive/issues/19



            IObservable <IObservable <BatchWriteRecord> > batches = _subject
                                                                    //
                                                                    // Batching
                                                                    //
                                                                    .Publish(connectedSource =>
            {
                var trigger = Observable.Merge(
                    // triggered by time & count
                    connectedSource.Window(TimeSpan.FromMilliseconds(
                                               writeOptions.FlushInterval),
                                           writeOptions.BatchSize,
                                           writeOptions.WriteScheduler),
                    // flush trigger
                    _flush
                    );
                return(connectedSource
                       .Window(trigger));
            })
                                                                    //
                                                                    // Group by key - same bucket, same org
                                                                    //
                                                                    .SelectMany(it => it.GroupBy(batchWrite => batchWrite.Options))
                                                                    //
                                                                    // Create Write Point = bucket, org, ... + data
                                                                    //
                                                                    .Select(grouped =>
            {
                var aggregate = grouped
                                .Aggregate(_stringBuilderPool.Get(), (builder, batchWrite) =>
                {
                    var data = batchWrite.ToLineProtocol();

                    if (string.IsNullOrEmpty(data))
                    {
                        return(builder);
                    }

                    if (builder.Length > 0)
                    {
                        builder.Append("\n");
                    }

                    return(builder.Append(data));
                }).Select(builder =>
                {
                    var result = builder.ToString();
                    builder.Clear();
                    _stringBuilderPool.Return(builder);
                    return(result);
                });

                return(aggregate.Select(records => new BatchWriteRecord(grouped.Key, records))
                       .Where(batchWriteItem => !string.IsNullOrEmpty(batchWriteItem.ToLineProtocol())));
            });

            if (writeOptions.JitterInterval > 0)
            {
                batches = batches
                          //
                          // Jitter
                          //
                          .Select(source =>
                {
                    return(source.Delay(_ => Observable.Timer(TimeSpan.FromMilliseconds(RetryAttempt.JitterDelay(writeOptions)), writeOptions.WriteScheduler)));
                });
            }
            var query = batches
                        .Concat()
                        //
                        // Map to Async request
                        //
                        .Select(batchWriteItem =>
            {
                var org          = batchWriteItem.Options.OrganizationId;
                var bucket       = batchWriteItem.Options.Bucket;
                var lineProtocol = batchWriteItem.ToLineProtocol();
                var precision    = batchWriteItem.Options.Precision;

                return(Observable
                       .Defer(() =>
                              service.PostWriteAsyncWithIRestResponse(org, bucket,
                                                                      Encoding.UTF8.GetBytes(lineProtocol), null,
                                                                      "identity", "text/plain; charset=utf-8", null, "application/json", null, precision)
                              .ToObservable())
                       .RetryWhen(f => f
                                  .Zip(Observable.Range(1, writeOptions.MaxRetries + 1), (exception, count)
                                       => new RetryAttempt(exception, count, writeOptions))
                                  .SelectMany(attempt =>
                {
                    if (attempt.IsRetry())
                    {
                        var retryInterval = attempt.GetRetryInterval();

                        var retryable = new WriteRetriableErrorEvent(org, bucket, precision, lineProtocol,
                                                                     attempt.Error, retryInterval);

                        Publish(retryable);

                        return Observable.Timer(TimeSpan.FromMilliseconds(retryInterval),
                                                writeOptions.WriteScheduler);
                    }

                    throw attempt.Error;
                }))
                       .Select(result =>
                {
                    // ReSharper disable once ConvertIfStatementToReturnStatement
                    if (result.IsSuccessful)
                    {
                        return Notification.CreateOnNext(result);
                    }

                    return Notification.CreateOnError <IRestResponse>(HttpException.Create(result, result.Content));
                })
                       .Catch <Notification <IRestResponse>, Exception>(ex =>
                {
                    var error = new WriteErrorEvent(org, bucket, precision, lineProtocol, ex);
                    Publish(error);

                    return Observable.Return(Notification.CreateOnError <IRestResponse>(ex));
                }).Do(res =>
                {
                    if (res.Kind == NotificationKind.OnNext)
                    {
                        var success = new WriteSuccessEvent(org, bucket, precision, lineProtocol);
                        Publish(success);
                    }
                }));
            })
                        .Concat()
                        .Subscribe(
                notification =>
            {
                switch (notification.Kind)
                {
                case NotificationKind.OnNext:
                    Trace.WriteLine($"The batch item: {notification} was processed successfully.");
                    break;

                case NotificationKind.OnError:
                    Trace.WriteLine(
                        $"The batch item wasn't processed successfully because: {notification.Exception}");
                    break;

                default:
                    Trace.WriteLine($"The batch item: {notification} was processed");
                    break;
                }
            },
                exception =>
            {
                _disposed = true;
                Trace.WriteLine($"The unhandled exception occurs: {exception}");
            },
                () =>
            {
                _disposed = true;
                Trace.WriteLine("The WriteApi was disposed.");
            });
        }