static string ReadRawPayload(int batchPostingLimit, long?eventBodyLimitBytes, ref FileSetPosition position, ref int count)
{
var payload = new StringWriter();
payload.Write("{\"Events\":[");
var delimStart = "";
using (var current = System.IO.File.Open(position.File !, FileMode.Open, FileAccess.Read, FileShare.ReadWrite))
{
var nextLineStart = position.NextLineStart;
while (count < batchPostingLimit && TryReadLine(current, ref nextLineStart, out var nextLine))
{
position = new FileSetPosition(nextLineStart, position.File);
// Count is the indicator that work was done, so advances even in the (rare) case an
// oversized event is dropped.
++count;
if (eventBodyLimitBytes.HasValue && Encoding.UTF8.GetByteCount(nextLine) > eventBodyLimitBytes.Value)
{
SelfLog.WriteLine(
"Event JSON representation exceeds the byte size limit of {0} and will be dropped; data: {1}",
eventBodyLimitBytes, nextLine);
}
else
{
payload.Write(delimStart);
payload.Write(nextLine);
delimStart = ",";
}
}
payload.Write("]}");
}
return(payload.ToString());
}
async Task OnTick()
{
try
{
int count;
do
{
count = 0;
// Locking the bookmark ensures that though there may be multiple instances of this
// class running, only one will ship logs at a time.
using (var bookmarkFile = _fileSet.OpenBookmarkFile())
{
var position = bookmarkFile.TryReadBookmark();
var files = _fileSet.GetFiles();
if (position.File == null || !IOFile.Exists(position.File))
{
position = new FileSetPosition(0, files.FirstOrDefault());
}
string payload;
if (position.File == null)
{
payload = PayloadReader.NoPayload;
count = 0;
}
else
{
payload = PayloadReader.ReadPayload(_batchPostingLimit, _eventBodyLimitBytes, ref position, ref count);
}
if (count > 0 || _controlledSwitch.IsActive && _nextRequiredLevelCheckUtc < DateTime.UtcNow)
{
_nextRequiredLevelCheckUtc = DateTime.UtcNow.Add(RequiredLevelCheckInterval);
var content = new StringContent(payload, Encoding.UTF8, "application/json");
if (!string.IsNullOrWhiteSpace(_apiKey))
{
content.Headers.Add(SeqApi.ApiKeyHeaderName, _apiKey);
}
var result = await _httpClient.PostAsync(SeqApi.BulkUploadResource, content).ConfigureAwait(false);
if (result.IsSuccessStatusCode)
{
_connectionSchedule.MarkSuccess();
bookmarkFile.WriteBookmark(position);
var returned = await result.Content.ReadAsStringAsync().ConfigureAwait(false);
var minimumAcceptedLevel = SeqApi.ReadEventInputResult(returned);
_controlledSwitch.Update(minimumAcceptedLevel);
}
else if (result.StatusCode == HttpStatusCode.BadRequest ||
result.StatusCode == HttpStatusCode.RequestEntityTooLarge)
{
// The connection attempt was successful - the payload we sent was the problem.
_connectionSchedule.MarkSuccess();
await DumpInvalidPayload(result, payload).ConfigureAwait(false);
bookmarkFile.WriteBookmark(position);
}
else
{
_connectionSchedule.MarkFailure();
SelfLog.WriteLine("Received failed HTTP shipping result {0}: {1}", result.StatusCode,
await result.Content.ReadAsStringAsync().ConfigureAwait(false));
break;
}
}
else if (position.File == null)
{
break;
}
else
{
// For whatever reason, there's nothing waiting to send. This means we should try connecting again at the
// regular interval, so mark the attempt as successful.
_connectionSchedule.MarkSuccess();
// Only advance the bookmark if no other process has the
// current file locked, and its length is as we found it.
if (files.Length == 2 && files.First() == position.File &&
FileIsUnlockedAndUnextended(position))
{
bookmarkFile.WriteBookmark(new FileSetPosition(0, files[1]));
}
if (files.Length > 2)
{
// Once there's a third file waiting to ship, we do our
// best to move on, though a lock on the current file
// will delay this.
IOFile.Delete(files[0]);
}
}
}
} while (count == _batchPostingLimit);
}
catch (Exception ex)
{
SelfLog.WriteLine("Exception while emitting periodic batch from {0}: {1}", this, ex);
_connectionSchedule.MarkFailure();
}
finally
{
lock (_stateLock)
{
if (!_unloading)
{
SetTimer();
}
}
}
}
