private ReductionPeriod LoadPeriod(ReductionSettings settings)
{
var storageKey = GetPositionStorageKey(settings);
DateTime?startDate;
try
{
startDate = _storage.Get <DateTime?>(storageKey);
Log.Trace("Loaded start date from storage: " + startDate);
}
catch (Exception)
{
startDate = settings.StartDate;
}
if (startDate == null)
{
startDate = DateTime.UtcNow.AddDays(-30);
}
var result = new ReductionPeriod();
result.From = new DateTime(startDate.Value.Year, startDate.Value.Month, startDate.Value.Day, 0, 0, 0, 0);
result.To = result.From.AddDays(1);
result.IsCurrent = startDate.Value.Date == DateTime.UtcNow.Date;
return(result);
}
private void ExecutePeriod()
{
var clientSettings =
new ConnectionSettings(
new Uri(string.Format("http://{0}:{1}", _reductionStructure.Settings.Host,
_reductionStructure.Settings.Port)));
clientSettings.SetDefaultIndex("_all");
_client = new ElasticClient(clientSettings);
ReductionPeriod period = LoadPeriod(_reductionStructure.Settings);
Log.Trace("Search type:" + typeof(TIn).Name);
var totalLogLines = new List <TIn>();
var position = 0;
while (true)
{
int currentPosition = position;
var logLines = _client.Search <TIn>(
s => s.Skip(currentPosition).Take(NumberOfDocumentsPerRead).Query(q =>
q.Range(r =>
r.OnField(ElasticSearchFields.Timestamp)
.From(period.From)
.To(period.To)
.ToExclusive()
)));
totalLogLines.AddRange(logLines.Documents);
if (logLines.Total == totalLogLines.Count)
{
break;
}
position += logLines.Documents.Count();
}
Log.Trace(totalLogLines.Count() + " log entries found between" + period.From.ToString("yyyy-MM-dd") + " and " + period.To.ToString("yyyy-MM-dd"));
var result = new Dictionary <string, ReductionResultData <TOut, THelp> >();
Parallel.ForEach(totalLogLines, () => new Dictionary <string, ReductionResultData <TOut, THelp> >(),
(record, loopControl, localDictionary) =>
{
_tokenSource.Token.ThrowIfCancellationRequested();
return(_reductionStructure.Reducer(record, localDictionary));
}
,
(localDictionary) =>
{
_tokenSource.Token.ThrowIfCancellationRequested();
lock (result)
{
_reductionStructure.Combiner(result, localDictionary);
}
});
EnsureIndexExists(_reductionStructure.Settings.IndexName);
foreach (var reductionResultData in result)
{
_client.Index(reductionResultData.Value, _indexName, GetType().Name, reductionResultData.Key);
_client.Flush();
}
if (period.IsCurrent)
{
Thread.Sleep(TimeSpan.FromSeconds(60));
return;
}
SetNewPeriod(_reductionStructure.Settings, period);
}
private void SetNewPeriod(ReductionSettings settings, ReductionPeriod reductionPeriod)
{
var storageKey = GetPositionStorageKey(settings);
_storage.Insert(storageKey, reductionPeriod.To);
}
