private void TryIngestMessages(IStreamLogIngestor ingestor, int limit, bool logMem = false)
{
string filename = "heapSize" + DateTime.Now.ToString("HH-mm-ss") + ".txt";
if(logMem)
File.WriteAllText(filename, "0");
for (int i = 0; i < limit; i++)
{
var inputEvent = new InputLogEvent();
inputEvent.ApplicationId = DataHelper.GetRandomApplication();
inputEvent.EventTime = DateTime.UtcNow;
inputEvent.Level = "ERROR";
inputEvent.Server = DataHelper.GetRandomServer();
inputEvent.Thread = "10";
inputEvent.Message = DataHelper.GetRandomError(); //_errors[0];
// ACT
var success = ingestor.TryIngest(inputEvent);
if (logMem && i%1000 == 0)
{
var mem = GC.GetTotalMemory(false);
File.AppendAllText(filename, "," + mem);
Console.WriteLine(i);
}
}
}
public void TryIngest_LargeVolumeOfEvents_ManualVerification()
{
// ARRANGE
var memory = new List<long>();
ContentFilterMocker.MockNotHigherThanApplicationCodeStackFrameFilter();
ConfigurationMocker.MockBucketSizeSeconds(60);
ConfigurationMocker.MockMaxPersistenceQueueSize(100000);
DataHelper.InitializeErrorData();
var ingestor = IngestorBuilder.CreateDefaultStreamIngestor(ConfigurationMocker.DefaultStreamIngestorConfigurationMock.Object, ContentFilterMocker.ContentFilterFactoryMock.Object);
int counter = 0;
while (counter < 10000)
{
var inputEvent = new InputLogEvent();
inputEvent.ApplicationId = DataHelper.GetRandomApplication();
inputEvent.EventTime = DateTime.UtcNow;
inputEvent.Level = "ERROR";
inputEvent.Server = DataHelper.GetRandomServer();
inputEvent.Thread = "10";
inputEvent.Message = DataHelper.GetRandomError();
// ACT
var success = ingestor.TryIngest(inputEvent);
Assert.AreEqual(true, success);
}
File.WriteAllText("heapSize.txt", string.Join(",", memory));
// ASSERT
Thread.Sleep(100000);
}
private bool ShouldRejectEvent(InputLogEvent inputLogEvent)
{
foreach (var eventFilter in _eventFilters)
{
if (eventFilter.ShouldReject(inputLogEvent.ApplicationId))
return true;
}
return false;
}
public ErrorEventEntity Parse(InputLogEvent inputLogEvent)
{
var error = new ErrorEventEntity();
error.Application = inputLogEvent.ApplicationId;
error.Server = inputLogEvent.Server;
error.MessageSource = inputLogEvent.Message;
ExtractExceptionData(error);
return error;
}
private void Reject(InputLogEvent inputLogEvent)
{
try
{
_eventRejecter.Reject(inputLogEvent);
}
catch (Exception)
{
// log it
}
}
public bool TryIngest(InputLogEvent inputLogEvent)
{
return Ingest(inputLogEvent);
//return await Task.Run(() => Ingest(inputLogEvent)).ConfigureAwait(false);
//Task<bool> task = Task.Run(() => Ingest(inputLogEvent));
//if (task == await Task.WhenAny(task, Task.Delay(timeoutMs)))
//{
// return await task;
//}
//throw new TimeoutException();
}
public ErrorEventEntity Parse(InputLogEvent inputLogEvent)
{
var error = new ErrorEventEntity();
error.Application = inputLogEvent.ApplicationId;
error.Server = inputLogEvent.Server;
error.MessageSource = inputLogEvent.Message;
var tokens = _errorTokenizer.Tokenize(inputLogEvent.Message).ToList();
var parseResult = new TokenBasedErrorParser(_grammarParserConfiguration).Parse(tokens);
error.ApplicationDateTime = parseResult.ErrorDateTime;
error.Exceptions = parseResult.Exceptions;
return error;
}
public void TryIngest_OneEvent_WithNotHigherStackFrameFilter_ManualVerification()
{
// ARRANGE
ContentFilterMocker.MockNotHigherThanApplicationCodeStackFrameFilter();
var inputEvent = new InputLogEvent();
inputEvent.ApplicationId = "ErrorGenerator";
inputEvent.EventTime = new DateTime(2016, 1, 28, 9, 0, 23, 997);
inputEvent.Level = "ERROR";
inputEvent.Server = "Gandalf";
inputEvent.Thread = "10";
inputEvent.Message = ErrorExamples.SpaceSeparated.TimeoutRemoting;
// ACT
var ingestor = IngestorBuilder.CreateDefaultStreamIngestor(ConfigurationMocker.DefaultStreamIngestorConfigurationMock.Object, ContentFilterMocker.ContentFilterFactoryMock.Object);
var success = ingestor.TryIngest(inputEvent);
// ASSERT
Assert.AreEqual(true, success);
Thread.Sleep(100000);
}
private void TryIngestMessages(IStreamLogIngestor ingestor, int limit)
{
for (int i = 0; i < limit; i++)
{
var inputEvent = new InputLogEvent();
inputEvent.ApplicationId = DataHelper.GetRandomApplication();
inputEvent.EventTime = DateTime.UtcNow;
inputEvent.Level = "ERROR";
inputEvent.Server = DataHelper.GetRandomServer();
inputEvent.Thread = "10";
inputEvent.Message = DataHelper.GetRandomError(); //_errors[0];
// ACT
var success = ingestor.TryIngest(inputEvent);
}
}
public void TryIngest_LargeVolumeOfEventsInParallel_ManualVerification()
{
// ARRANGE
var memory = new List<long>();
ContentFilterMocker.MockNotHigherThanApplicationCodeStackFrameFilter();
ConfigurationMocker.MockBucketSizeSeconds(60);
ConfigurationMocker.MockMaxPersistenceQueueSize(1000000);
DataHelper.InitializeErrorData();
var ingestor = IngestorBuilder.CreateDefaultStreamIngestor(ConfigurationMocker.DefaultStreamIngestorConfigurationMock.Object, ContentFilterMocker.ContentFilterFactoryMock.Object);
object syncRoot = new object();
int counter = 0;
int limit = 500000;
Task p = Task.Run(() =>
Parallel.For(0, limit, i =>
{
var inputEvent = new InputLogEvent();
inputEvent.ApplicationId = DataHelper.GetRandomApplication();
inputEvent.EventTime = DateTime.UtcNow;
inputEvent.Level = "ERROR";
inputEvent.Server = DataHelper.GetRandomServer();
inputEvent.Thread = "10";
inputEvent.Message = DataHelper.GetRandomError(); //_errors[0];
// ACT
var success = ingestor.TryIngest(inputEvent);
})
);
Task m = Task.Run(() => LogTotalMemory());
Task.WaitAll(new[] {p, m});
// ASSERT
}
public ErrorEventPacket(InputLogEvent inputLogEvent)
{
InputLogEvent = inputLogEvent;
}
private ErrorEventPacket RecombineEventPacket(ErrorDefinitionEntity errorDefinition, ErrorInstanceEntity errorInstance)
{
var inputLogEvent = new InputLogEvent();
inputLogEvent.Message = errorDefinition.MessageSource;
var errorEvent = new ErrorEventEntity();
errorEvent.Application = errorDefinition.Application;
errorEvent.ApplicationDateTime = errorInstance.ApplicationDateTime;
errorEvent.Exceptions = RecombineExceptions(errorDefinition.Exceptions, errorInstance.Exceptions);
errorEvent.Fingerprint = errorDefinition.Fingerprint;
errorEvent.MessageSource = errorDefinition.MessageSource;
errorEvent.Server = GetFromServerCache(errorInstance.ServerCacheId);
var eventPacket = new ErrorEventPacket(inputLogEvent);
eventPacket.ErrorEventEntity = errorEvent;
return eventPacket;
}
public async Task RejectAsync(InputLogEvent inputLogEvent)
{
Task t = Task.Run(() => true);
await t;
}
public void Reject(InputLogEvent inputLogEvent)
{
}
private bool Ingest(InputLogEvent inputLogEvent)
{
var errorEventPacket = new ErrorEventPacket(inputLogEvent);
return _firstPipelineStage.Process(errorEventPacket);
}
