public void Process(Configuration.source source)
{
ILogPersister logPersister = _persisterFactory.GetPersister(source.destination.mongoConnectionString,
source.destination.mongoDatabase,
source.destination.mongoCollection);
logPersister.BatchSize = source.batchSize.GetValueOrDefault();
LogReaderState state = _stateRecorder[source.id];
IEnumerable<FileInfo> files = _logFetcher.GetLogFiles(source.logDirectory,
state.IsNotNull() ? state.lastFile : null,
source.logAll.GetValueOrDefault());
if (state.IsNotNull())
{
if (files.Any(x => x.Name == state.lastFile))
{
W3CLogReader w3c = _readerFactory.CreateLogReader(source);
w3c.FromLine(state.lastLine);
w3c.WithFields(state.fields);
_logger.Log("Begine File:\"{0}\" at Line:{1}".Frmat(files.First().FullName, state.lastLine));
logPersister.Persist(w3c.Process(_logFetcher.GetLines(files.First())));
_stateRecorder[source.id] = new LogReaderState
{
lastFile = files.First().Name,
lastLine = w3c.LastLine,
fields = w3c.Fields.ToArray()
};
files = files.Skip(1).ToList();
}
}
files.ForEach(file =>
{
W3CLogReader w3c = _readerFactory.CreateLogReader(source);
_logger.Log("Begine File:\"{0}\" from line 0".Frmat(file.FullName));
IEnumerable<IDictionary<string, object>> wtf = w3c.Process(_logFetcher.GetLines(file));
logPersister.Persist(wtf);
_stateRecorder[source.id] = new LogReaderState
{
lastFile = file.Name,
lastLine = w3c.LastLine,
fields = w3c.Fields.ToArray()
};
});
}
public void Setup()
{
config = new Configuration();
;
source = new Configuration.source();
source.aliases.Add(new Configuration.source.elementAlias {alias = "theAlias", elementName = "theelemtnname"});
source.conversions.Add(new Configuration.source.conversion {elementName = "LogTime", type = "DateTime"});
source.composites.Add(new Configuration.source.composite
{elementName = "LogTime", elements = new[] {"date", "time"}});
source.multiColumnElements.Add(new Configuration.source.mulitColumnElement
{elementName = "Endtime-UTC", colSpan = 2});
source.staticElements.Add(new Configuration.source.staticElement
{elementName = "MyName", elementValue = "Dillon"});
source.drop.Add("el1");
source.drop.Add("el2");
source.entryLimit = 100;
config.sources.Add(source);
cl = new W3CLogReaderFactory(Substitute.For<Logger>());
}