protected internal WriteApi(InfluxDBClientOptions options, WriteService service, WriteOptions writeOptions,
InfluxDBClient influxDbClient)
{
Arguments.CheckNotNull(service, nameof(service));
Arguments.CheckNotNull(writeOptions, nameof(writeOptions));
Arguments.CheckNotNull(influxDbClient, nameof(_influxDbClient));
_options = options;
_influxDbClient = influxDbClient;
// backpreasure - is not implemented in C#
//
// => use unbound buffer
//
// https://github.com/dotnet/reactive/issues/19
var observable = _subject.ObserveOn(writeOptions.WriteScheduler);
var boundary = observable
.Buffer(TimeSpan.FromMilliseconds(writeOptions.FlushInterval), writeOptions.BatchSize,
writeOptions.WriteScheduler)
.Merge(_flush);
observable
//
// Batching
//
.Window(boundary)
//
// Group by key - same bucket, same org
//
.SelectMany(it => it.GroupBy(batchWrite => batchWrite.Options))
//
// Create Write Point = bucket, org, ... + data
//
.Select(grouped =>
{
var aggregate = grouped
.Aggregate(new StringBuilder(""), (builder, batchWrite) =>
{
var data = batchWrite.ToLineProtocol();
if (string.IsNullOrEmpty(data))
{
return(builder);
}
if (builder.Length > 0)
{
builder.Append("\n");
}
return(builder.Append(data));
}).Select(builder => builder.ToString());
return(aggregate.Select(records => new BatchWriteRecord(grouped.Key, records)));
})
//
// Jitter
//
.Select(source =>
{
if (writeOptions.JitterInterval <= 0)
{
return(source);
}
return(source.Delay(_ => Observable.Timer(TimeSpan.FromMilliseconds(JitterDelay(writeOptions)), Scheduler.CurrentThread)));
})
.Concat()
//
// Map to Async request
//
.Where(batchWriteItem => !string.IsNullOrEmpty(batchWriteItem.ToLineProtocol()))
.Select(batchWriteItem =>
{
var org = batchWriteItem.Options.OrganizationId;
var bucket = batchWriteItem.Options.Bucket;
var lineProtocol = batchWriteItem.ToLineProtocol();
var precision = batchWriteItem.Options.Precision;
return(Observable
.Defer(() =>
service.PostWriteAsyncWithIRestResponse(org, bucket,
Encoding.UTF8.GetBytes(lineProtocol), null,
"identity", "text/plain; charset=utf-8", null, "application/json", null, precision)
.ToObservable())
.RetryWhen(f => f.SelectMany(e =>
{
if (e is HttpException httpException)
{
//
// This types is not able to retry
//
if (httpException.Status != 429 && httpException.Status != 503)
{
throw httpException;
}
var retryInterval = (httpException.RetryAfter * 1000 ?? writeOptions.RetryInterval) +
JitterDelay(writeOptions);
var retryable = new WriteRetriableErrorEvent(org, bucket, precision, lineProtocol,
httpException, retryInterval);
Publish(retryable);
return Observable.Timer(TimeSpan.FromMilliseconds(retryInterval));
}
throw e;
}))
.Select(result =>
{
// ReSharper disable once ConvertIfStatementToReturnStatement
if (result.IsSuccessful)
{
return Notification.CreateOnNext(result);
}
return Notification.CreateOnError <IRestResponse>(HttpException.Create(result));
})
.Catch <Notification <IRestResponse>, Exception>(ex =>
{
var error = new WriteErrorEvent(org, bucket, precision, lineProtocol, ex);
Publish(error);
return Observable.Return(Notification.CreateOnError <IRestResponse>(ex));
}).Do(res =>
{
if (res.Kind == NotificationKind.OnNext)
{
var success = new WriteSuccessEvent(org, bucket, precision, lineProtocol);
Publish(success);
}
}));
})
.Concat()
.Subscribe(
notification =>
{
switch (notification.Kind)
{
case NotificationKind.OnNext:
Trace.WriteLine($"The batch item: {notification} was processed successfully.");
break;
case NotificationKind.OnError:
Trace.WriteLine(
$"The batch item wasn't processed successfully because: {notification.Exception}");
break;
default:
Trace.WriteLine($"The batch item: {notification} was processed");
break;
}
},
exception => Trace.WriteLine($"The unhandled exception occurs: {exception}"),
() => Trace.WriteLine("The WriteApi was disposed."));
}
protected internal WriteApi(
InfluxDBClientOptions options,
WriteService service,
WriteOptions writeOptions,
IDomainObjectMapper mapper,
InfluxDBClient influxDbClient,
IObservable <Unit> disposeCommand)
{
Arguments.CheckNotNull(service, nameof(service));
Arguments.CheckNotNull(writeOptions, nameof(writeOptions));
Arguments.CheckNotNull(mapper, nameof(mapper));
Arguments.CheckNotNull(influxDbClient, nameof(_influxDbClient));
Arguments.CheckNotNull(disposeCommand, nameof(disposeCommand));
_options = options;
_mapper = mapper;
_influxDbClient = influxDbClient;
_unsubscribeDisposeCommand = disposeCommand.Subscribe(_ => Dispose());
// backpreasure - is not implemented in C#
//
// => use unbound buffer
//
// https://github.com/dotnet/reactive/issues/19
IObservable <IObservable <BatchWriteRecord> > batches = _subject
//
// Batching
//
.Publish(connectedSource =>
{
var trigger = Observable.Merge(
// triggered by time & count
connectedSource.Window(TimeSpan.FromMilliseconds(
writeOptions.FlushInterval),
writeOptions.BatchSize,
writeOptions.WriteScheduler),
// flush trigger
_flush
);
return(connectedSource
.Window(trigger));
})
//
// Group by key - same bucket, same org
//
.SelectMany(it => it.GroupBy(batchWrite => batchWrite.Options))
//
// Create Write Point = bucket, org, ... + data
//
.Select(grouped =>
{
var aggregate = grouped
.Aggregate(_stringBuilderPool.Get(), (builder, batchWrite) =>
{
var data = batchWrite.ToLineProtocol();
if (string.IsNullOrEmpty(data))
{
return(builder);
}
if (builder.Length > 0)
{
builder.Append("\n");
}
return(builder.Append(data));
}).Select(builder =>
{
var result = builder.ToString();
builder.Clear();
_stringBuilderPool.Return(builder);
return(result);
});
return(aggregate.Select(records => new BatchWriteRecord(grouped.Key, records))
.Where(batchWriteItem => !string.IsNullOrEmpty(batchWriteItem.ToLineProtocol())));
});
if (writeOptions.JitterInterval > 0)
{
batches = batches
//
// Jitter
//
.Select(source =>
{
return(source.Delay(_ => Observable.Timer(TimeSpan.FromMilliseconds(RetryAttempt.JitterDelay(writeOptions)), writeOptions.WriteScheduler)));
});
}
var query = batches
.Concat()
//
// Map to Async request
//
.Select(batchWriteItem =>
{
var org = batchWriteItem.Options.OrganizationId;
var bucket = batchWriteItem.Options.Bucket;
var lineProtocol = batchWriteItem.ToLineProtocol();
var precision = batchWriteItem.Options.Precision;
return(Observable
.Defer(() =>
service.PostWriteAsyncWithIRestResponse(org, bucket,
Encoding.UTF8.GetBytes(lineProtocol), null,
"identity", "text/plain; charset=utf-8", null, "application/json", null, precision)
.ToObservable())
.RetryWhen(f => f
.Zip(Observable.Range(1, writeOptions.MaxRetries + 1), (exception, count)
=> new RetryAttempt(exception, count, writeOptions))
.SelectMany(attempt =>
{
if (attempt.IsRetry())
{
var retryInterval = attempt.GetRetryInterval();
var retryable = new WriteRetriableErrorEvent(org, bucket, precision, lineProtocol,
attempt.Error, retryInterval);
Publish(retryable);
return Observable.Timer(TimeSpan.FromMilliseconds(retryInterval),
writeOptions.WriteScheduler);
}
throw attempt.Error;
}))
.Select(result =>
{
// ReSharper disable once ConvertIfStatementToReturnStatement
if (result.IsSuccessful)
{
return Notification.CreateOnNext(result);
}
return Notification.CreateOnError <IRestResponse>(HttpException.Create(result, result.Content));
})
.Catch <Notification <IRestResponse>, Exception>(ex =>
{
var error = new WriteErrorEvent(org, bucket, precision, lineProtocol, ex);
Publish(error);
return Observable.Return(Notification.CreateOnError <IRestResponse>(ex));
}).Do(res =>
{
if (res.Kind == NotificationKind.OnNext)
{
var success = new WriteSuccessEvent(org, bucket, precision, lineProtocol);
Publish(success);
}
}));
})
.Concat()
.Subscribe(
notification =>
{
switch (notification.Kind)
{
case NotificationKind.OnNext:
Trace.WriteLine($"The batch item: {notification} was processed successfully.");
break;
case NotificationKind.OnError:
Trace.WriteLine(
$"The batch item wasn't processed successfully because: {notification.Exception}");
break;
default:
Trace.WriteLine($"The batch item: {notification} was processed");
break;
}
},
exception =>
{
_disposed = true;
Trace.WriteLine($"The unhandled exception occurs: {exception}");
},
() =>
{
_disposed = true;
Trace.WriteLine("The WriteApi was disposed.");
});
}