protected override int LoadInternal(IEnumerable <ElasticSearchIndexWithRecords> records, DocumentsOperationContext context, EtlStatsScope scope)
{
int count = 0;
_client ??= ElasticSearchHelper.CreateClient(Configuration.Connection);
foreach (var index in records)
{
string indexName = index.IndexName.ToLower();
EnsureIndexExistsAndValidateIfNeeded(indexName, index);
if (index.InsertOnlyMode == false)
{
count += DeleteByQueryOnIndexIdProperty(index);
}
if (index.Inserts.Count > 0)
{
var streamHandler = PostData.StreamHandler(index.Inserts, (inserts, stream) =>
{
foreach (ElasticSearchItem insert in inserts)
{
if (insert.TransformationResult == null)
{
continue;
}
stream.Write(IndexBulkActionBytes);
using (var json = EnsureLowerCasedIndexIdProperty(context, insert.TransformationResult, index))
using (var writer = new BlittableJsonTextWriter(context, stream))
{
writer.WriteNewLine();
writer.WriteObject(json);
writer.WriteNewLine();
}
count++;
}
}, (i, s, token) => throw new NotSupportedException("We don't use async bulk method"));
var bulkIndexResponse = _client.LowLevel.Bulk <BulkResponse>(indexName, streamHandler, new BulkRequestParameters {
Refresh = Refresh.WaitFor
});
if (bulkIndexResponse.IsValid == false)
{
ThrowElasticSearchLoadException($"Failed to index data to '{indexName}' index", bulkIndexResponse.ServerError, bulkIndexResponse.OriginalException,
bulkIndexResponse.DebugInformation);
}
}
}
return(count);
}
public async Task StartSendingNotifications(bool sendStartTime, bool throttleConnection)
{
if (sendStartTime)
{
SendStartTime();
}
JsonOperationContext context;
using (_documentDatabase.DocumentsStorage.ContextPool.AllocateOperationContext(out context))
{
using (var ms = new MemoryStream())
{
var sp = Stopwatch.StartNew();
while (true)
{
if (_disposeToken.IsCancellationRequested)
{
break;
}
ms.SetLength(0);
using (var writer = new BlittableJsonTextWriter(context, ms))
{
sp.Restart();
do
{
var value = await GetNextMessage(throttleConnection);
if (_disposeToken.IsCancellationRequested)
{
break;
}
if (value == null)
{
break;
}
context.Write(writer, value);
writer.WriteNewLine();
if (ms.Length > 16 * 1024)
{
break;
}
} while (_sendQueue.Count > 0 && sp.Elapsed < TimeSpan.FromSeconds(5));
}
if (ms.Length == 0)
{
// ensure that we send _something_ over the network, to keep the
// connection alive
ms.WriteByte((byte)'\r');
ms.WriteByte((byte)'\n');
}
ArraySegment <byte> bytes;
ms.TryGetBuffer(out bytes);
await _webSocket.SendAsync(bytes, WebSocketMessageType.Text, true, _disposeToken.Token);
}
}
}
}