public void Process(Configuration.source source) { ILogPersister logPersister = _persisterFactory.GetPersister(source.destination.mongoConnectionString, source.destination.mongoDatabase, source.destination.mongoCollection); logPersister.BatchSize = source.batchSize.GetValueOrDefault(); LogReaderState state = _stateRecorder[source.id]; IEnumerable<FileInfo> files = _logFetcher.GetLogFiles(source.logDirectory, state.IsNotNull() ? state.lastFile : null, source.logAll.GetValueOrDefault()); if (state.IsNotNull()) { if (files.Any(x => x.Name == state.lastFile)) { W3CLogReader w3c = _readerFactory.CreateLogReader(source); w3c.FromLine(state.lastLine); w3c.WithFields(state.fields); _logger.Log("Begine File:\"{0}\" at Line:{1}".Frmat(files.First().FullName, state.lastLine)); logPersister.Persist(w3c.Process(_logFetcher.GetLines(files.First()))); _stateRecorder[source.id] = new LogReaderState { lastFile = files.First().Name, lastLine = w3c.LastLine, fields = w3c.Fields.ToArray() }; files = files.Skip(1).ToList(); } } files.ForEach(file => { W3CLogReader w3c = _readerFactory.CreateLogReader(source); _logger.Log("Begine File:\"{0}\" from line 0".Frmat(file.FullName)); IEnumerable<IDictionary<string, object>> wtf = w3c.Process(_logFetcher.GetLines(file)); logPersister.Persist(wtf); _stateRecorder[source.id] = new LogReaderState { lastFile = file.Name, lastLine = w3c.LastLine, fields = w3c.Fields.ToArray() }; }); }
public void Setup() { config = new Configuration(); ; source = new Configuration.source(); source.aliases.Add(new Configuration.source.elementAlias {alias = "theAlias", elementName = "theelemtnname"}); source.conversions.Add(new Configuration.source.conversion {elementName = "LogTime", type = "DateTime"}); source.composites.Add(new Configuration.source.composite {elementName = "LogTime", elements = new[] {"date", "time"}}); source.multiColumnElements.Add(new Configuration.source.mulitColumnElement {elementName = "Endtime-UTC", colSpan = 2}); source.staticElements.Add(new Configuration.source.staticElement {elementName = "MyName", elementValue = "Dillon"}); source.drop.Add("el1"); source.drop.Add("el2"); source.entryLimit = 100; config.sources.Add(source); cl = new W3CLogReaderFactory(Substitute.For<Logger>()); }