public void SimpleTest()
{
LogEntityFactory logEntityFactory = new LogEntityFactory();
LogEntity logEntity = logEntityFactory.GetRandomLogEntity();
ILogParser logParser = new CommonLogParser();
DateTime now = DateTime.UtcNow;
try
{
var response = logParser.Parse(logEntity.ToString(), now);
Assert.AreEqual(logEntity.IpAddress, response.IpAddress, "ip address");
Assert.AreEqual(logEntity.UserId, response.UserId, "user id");
Assert.AreEqual(logEntity.ClientIdentity, response.ClientIdentity, "client identity");
Assert.AreEqual(logEntity.Time.ToShortTimeString(), response.Time.ToShortTimeString(), "time");
Assert.AreEqual(logEntity.HttpMethod, response.HttpMethod, "http method");
Assert.AreEqual(logEntity.HttpPath, response.HttpPath, "http path");
Assert.AreEqual(logEntity.HttpVersion, response.HttpVersion, "http version");
Assert.AreEqual(logEntity.StatusCode, response.StatusCode, "status code");
Assert.AreEqual(logEntity.ResponseSize, response.ResponseSize, "size");
Assert.AreEqual(now, response.TimeReadFromFile, "read from file");
}
catch (Exception ex)
{
Assert.Fail(ex.ToString());
}
}
static async Task Main(string[] args)
{
//
// Hi! Welcome to my assignment, I am Catalin David - nice to (virtually) meet you!
//
// Disclaimers:
// * I understand that DataDog is a Python / Go / other non-.NET languages, so I appreciate taking
// the time to try to read through my C# code.
// * I am making extensive use of async/await and Task/Task<T> throughout the code.
// These concepts are similar to Go sub-routines (the runtime takes care of creating the necessary
// threads and scheduling these items on those threads), Promises (in JavaScript).
// I found this doc to be rather informative: https://docs.microsoft.com/en-us/dotnet/csharp/async
// * There are 3rd party libraries for .NET that implement time sliding windows, but I have considered
// that the intent of this assignment is not to test my knowledge of .NET libraries, but to actually
// implement something by myself, so I have decided to not use such libraries.
// That being said, if I were to write production code, I would probably not be using the current
// implementation. Instead, I would be using RX (Reactive eXtensions): https://github.com/Reactive-Extensions/Rx.NET
// This library makes it easy to operate using the Observable pattern and a time sliding window with
// events accummulated can be implemented using Observable.Window (to define the time window),
// Observable.Interval (to trigger the reporting) and an IFileReader (under Interfaces folder) that
// would generate the observable events. See an example of observable window here:
// http://rxwiki.wikidot.com/101samples#toc38
//
//
// Now that we are done with the disclaimers, please allow me to introduce you to my solution.
//
// General notes:
// * I am testing this using the LogWriter project - this outputs data to a well-known file (~\datadog\file.log)
// in the common log format.
// * I try to use interfaces in most places as that makes it easy to unit test different components and mock out everything else.
// * In production, we would use Dependency Injection (DI) to construct the objects and their dependencies, but I decided to not
// use that here (here we instantiate the objects manually).
// * There are very many tests missing. So far, I have added one because RegEx-s scare me and I wanted to make sure that
// I got it right. There are tests missing for the log parser and the logic itself.
// * To run, you need the following:
// 1) Download https://github.com/dotnet/core/blob/master/release-notes/download-archives/2.0.0-download.md
// 2) cd to LogWriter -> dotnet run (to start writing into ~\datadog\file.log)
// 3) cd to LogReader -> dotnet run (this will start reading from the logfile in ~\datadog\file.log if it exists)
//
//
// The structure of this project (the LogReader) is as follows:
// * we have a reader flow (Task) that run continuously - this reads the file and generates the necessary data.
// * we have a reporting flow (Task) that runs every Interval and generates the report as output.
//
// THE READER FLOW
// * we have an IFileReader with the implementation FileReader - this tries to read as fast as possible strings from
// the log file and pump the strings to an asynchronous FIFO queue (BufferBlock).
// * .NET provides a library called TPL Dataflow (TPL = Task Parallel Library) that provides building "blocks"
// for data flows (these seem very similar to Go channels). In particular, I am leveraging here a BufferBlock
// which is an async FIFO queue. This allows us to have a producer / consumer model where the FileReader is the producer
// that can go back to reading without caring when the log line is parsed.
// This document explains the blocks that I am using:
// https://docs.microsoft.com/en-us/dotnet/standard/parallel-programming/dataflow-task-parallel-library#predefined-dataflow-block-types
// * Next, we have another Dataflow block: the TransformBlock. This block takes the output of the FileReader and parses it
// to a C# object that represents the log event. This block is chained to the BufferBlock.
// * Next, we have an Action block. This is a builtin block that is chained to the TransformBlock above.
// This block executes a method on a block - in our case, it adds the LogEvent to the IReportGenerator.
//
// THE REPORTING FLOW
// * Every interval, we call into the IReportGenerator to do the following:
// 1) generate a summary report for the last interval (10 seconds)
// 2) generate an alerting report for the last 2 minutes.
CancellationTokenSource cts = new CancellationTokenSource();
LoggerFactory = new LoggerFactory().AddConsole(LogLevel.Information);
ILogger logger = LoggerFactory.CreateLogger(nameof(Program));
// The services that will do the work
ILogParser logParser = new CommonLogParser();
IReportGenerator reportGenerator = new TenSecondTwoMinuteReportGenerator(LoggerFactory.CreateLogger <TenSecondTwoMinuteReportGenerator>());
// Setting up the TPL Dataflow
var bufferBlock = new BufferBlock <Tuple <string, DateTime> >();
var transformBlock = new TransformBlock <Tuple <string, DateTime>, LogEntity>(input =>
{
return(logParser.Parse(input.Item1, input.Item2));
});
var actionBlock = new ActionBlock <LogEntity>(input =>
{
reportGenerator.AddLogItem(input);
});
bufferBlock.LinkTo(transformBlock); // we chain the buffer block to the transform block
transformBlock.LinkTo(actionBlock); // we chain the transform block to the action block
// Done setting up TPL
IFileReader fileReader = new FileReader(LogSettings.FILE_PATH, bufferBlock);
Task readingTask = fileReader.StartReadAsync(cts.Token);
Task reportingTask = reportGenerator.StartPeriodicReportingAsync(cts.Token);
bool toContinue = true;
logger.LogError("Press x or X to eXit");
while (toContinue)
{
var key = Console.ReadKey();
switch (key.KeyChar)
{
case 'x':
case 'X':
toContinue = false;
cts.Cancel();
logger.LogWarning("Cancelled the tasks...");
break;
default:
logger.LogError("Press x or X to eXit");
break;
}
}
try
{
await Task.WhenAll(readingTask, reportingTask);
}
catch (Exception)
{
}
}