public void GivingTheLastSeenEventKeyRemovesPrecedingEvents()
{
using (var temp = TempFolder.ForCaller())
using (var buffer = new LogBuffer(temp.AllocateFilename("mdb"), DefaultBufferSize))
{
byte[] a1 = Some.Bytes(140), a2 = Some.Bytes(140), a3 = Some.Bytes(140);
buffer.Enqueue(new[] { a1, a2, a3 });
var contents = buffer.Peek(420);
Assert.Equal(3, contents.Length);
buffer.Dequeue(contents[2].Key);
var remaining = buffer.Peek(420);
Assert.Equal(0, remaining.Length);
}
}
void OnTick()
{
try
{
var sendingSingles = 0;
do
{
var available = _logBuffer.Peek((int)_outputConfig.RawPayloadLimitBytes);
if (available.Length == 0)
{
// For whatever reason, there's nothing waiting to send. This means we should try connecting again at the
// regular interval, so mark the attempt as successful.
_connectionSchedule.MarkSuccess();
break;
}
Stream payload;
ulong lastIncluded;
MakePayload(available, sendingSingles > 0, out payload, out lastIncluded);
var content = new StreamContent(new UnclosableStreamWrapper(payload));
content.Headers.ContentType = new MediaTypeHeaderValue("application/json")
{
CharSet = Encoding.UTF8.WebName
};
if (!string.IsNullOrWhiteSpace(_outputConfig.ApiKey))
{
content.Headers.Add(ApiKeyHeaderName, _outputConfig.ApiKey);
}
var result = _httpClient.PostAsync(BulkUploadResource, content).Result;
if (result.IsSuccessStatusCode)
{
_connectionSchedule.MarkSuccess();
_logBuffer.Dequeue(lastIncluded);
if (sendingSingles > 0)
{
sendingSingles--;
}
}
else if (result.StatusCode == HttpStatusCode.BadRequest ||
result.StatusCode == HttpStatusCode.RequestEntityTooLarge)
{
// The connection attempt was successful - the payload we sent was the problem.
_connectionSchedule.MarkSuccess();
if (sendingSingles != 0)
{
payload.Position = 0;
var payloadText = new StreamReader(payload, Encoding.UTF8).ReadToEnd();
Log.Error("HTTP shipping failed with {StatusCode}: {Result}; payload was {InvalidPayload}", result.StatusCode, result.Content.ReadAsStringAsync().Result, payloadText);
_logBuffer.Dequeue(lastIncluded);
sendingSingles = 0;
}
else
{
// Unscientific (shoudl "binary search" in batches) but sending the next
// hundred events singly should flush out the problematic one.
sendingSingles = 100;
}
}
else
{
_connectionSchedule.MarkFailure();
Log.Error("Received failed HTTP shipping result {StatusCode}: {Result}", result.StatusCode, result.Content.ReadAsStringAsync().Result);
break;
}
}while (true);
}
catch (Exception ex)
{
Log.Error(ex, "Exception while sending a batch from the log shipper");
_connectionSchedule.MarkFailure();
}
finally
{
lock (_stateLock)
{
if (!_unloading)
{
SetTimer();
}
}
}
}