private ReductionPeriod LoadPeriod(ReductionSettings settings) { var storageKey = GetPositionStorageKey(settings); DateTime?startDate; try { startDate = _storage.Get <DateTime?>(storageKey); Log.Trace("Loaded start date from storage: " + startDate); } catch (Exception) { startDate = settings.StartDate; } if (startDate == null) { startDate = DateTime.UtcNow.AddDays(-30); } var result = new ReductionPeriod(); result.From = new DateTime(startDate.Value.Year, startDate.Value.Month, startDate.Value.Day, 0, 0, 0, 0); result.To = result.From.AddDays(1); result.IsCurrent = startDate.Value.Date == DateTime.UtcNow.Date; return(result); }
private void ExecutePeriod() { var clientSettings = new ConnectionSettings( new Uri(string.Format("http://{0}:{1}", _reductionStructure.Settings.Host, _reductionStructure.Settings.Port))); clientSettings.SetDefaultIndex("_all"); _client = new ElasticClient(clientSettings); ReductionPeriod period = LoadPeriod(_reductionStructure.Settings); Log.Trace("Search type:" + typeof(TIn).Name); var totalLogLines = new List <TIn>(); var position = 0; while (true) { int currentPosition = position; var logLines = _client.Search <TIn>( s => s.Skip(currentPosition).Take(NumberOfDocumentsPerRead).Query(q => q.Range(r => r.OnField(ElasticSearchFields.Timestamp) .From(period.From) .To(period.To) .ToExclusive() ))); totalLogLines.AddRange(logLines.Documents); if (logLines.Total == totalLogLines.Count) { break; } position += logLines.Documents.Count(); } Log.Trace(totalLogLines.Count() + " log entries found between" + period.From.ToString("yyyy-MM-dd") + " and " + period.To.ToString("yyyy-MM-dd")); var result = new Dictionary <string, ReductionResultData <TOut, THelp> >(); Parallel.ForEach(totalLogLines, () => new Dictionary <string, ReductionResultData <TOut, THelp> >(), (record, loopControl, localDictionary) => { _tokenSource.Token.ThrowIfCancellationRequested(); return(_reductionStructure.Reducer(record, localDictionary)); } , (localDictionary) => { _tokenSource.Token.ThrowIfCancellationRequested(); lock (result) { _reductionStructure.Combiner(result, localDictionary); } }); EnsureIndexExists(_reductionStructure.Settings.IndexName); foreach (var reductionResultData in result) { _client.Index(reductionResultData.Value, _indexName, GetType().Name, reductionResultData.Key); _client.Flush(); } if (period.IsCurrent) { Thread.Sleep(TimeSpan.FromSeconds(60)); return; } SetNewPeriod(_reductionStructure.Settings, period); }
private void SetNewPeriod(ReductionSettings settings, ReductionPeriod reductionPeriod) { var storageKey = GetPositionStorageKey(settings); _storage.Insert(storageKey, reductionPeriod.To); }