public void If_execution_takes_longer_than_period_it_triggers_next_execution_immediately_after_previous()
{
var counter = 0;
var failure = false;
var lastEndTime = DateTime.MinValue;
var @event = new ManualResetEventSlim(false);
var delay = TimeSpan.Zero;
var executor = new PeriodicExecutor(() =>
{
delay = DateTime.Now - lastEndTime;
if (lastEndTime != DateTime.MinValue && delay > TimeSpan.FromMilliseconds(100))
{
@event.Set();
failure = true;
return;
}
counter++;
Thread.Sleep(2000);
lastEndTime = DateTime.Now;
if (counter == 2)
{
@event.Set();
}
}, TimeSpan.FromSeconds(1));
executor.Start(true);
@event.Wait();
executor.Stop(CancellationToken.None);
Assert.IsFalse(failure, string.Format("Time between finishing previous execution and starting this longer than {0} ms",delay));
}
public void Can_shutdown_while_waiting()
{
var @event = new ManualResetEventSlim(false);
var executor = new PeriodicExecutor(@event.Set, TimeSpan.FromSeconds(10000));
executor.Start(false);
@event.Wait();
Thread.Sleep(1000);
executor.Stop(CancellationToken.None);
Assert.Pass();
}
public void Execute(DocumentDatabase database)
{
Database = database;
indexName = new ExpiryProcessedMessageIndex().IndexName;
deletionBatchSize = Settings.ExpirationProcessBatchSize;
deleteFrequencyInSeconds = Settings.ExpirationProcessTimerInSeconds;
if (deleteFrequencyInSeconds == 0)
{
return;
}
logger.Info("Expired Documents every {0} seconds", deleteFrequencyInSeconds);
logger.Info("Deletion Batch Size: {0}", deletionBatchSize);
logger.Info("Retention Period: {0}", Settings.HoursToKeepMessagesBeforeExpiring);
timer = new PeriodicExecutor(Delete,TimeSpan.FromSeconds(deleteFrequencyInSeconds));
timer.Start(true);
}
public void Execute(DocumentDatabase database)
{
Database = database;
indexName = new ExpiryProcessedMessageIndex().IndexName;
deletionBatchSize = Settings.ExpirationProcessBatchSize;
deleteFrequencyInSeconds = Settings.ExpirationProcessTimerInSeconds;
if (deleteFrequencyInSeconds == 0)
{
return;
}
logger.Info("Expired Documents every {0} seconds", deleteFrequencyInSeconds);
logger.Info("Deletion Batch Size: {0}", deletionBatchSize);
logger.Info("Retention Period: {0}", Settings.HoursToKeepMessagesBeforeExpiring);
timer = new PeriodicExecutor(Delete, TimeSpan.FromSeconds(deleteFrequencyInSeconds));
timer.Start(true);
}
public void If_execution_throws_it_does_not_kill_the_executor()
{
var first = true;
var success = false;
var @event = new ManualResetEventSlim(false);
var executor = new PeriodicExecutor(() =>
{
if (first)
{
first = false;
throw new Exception();
}
success = true;
@event.Set();
}, TimeSpan.FromSeconds(1));
executor.Start(true);
@event.Wait();
executor.Stop(CancellationToken.None);
Assert.IsTrue(success);
}
void Delete(PeriodicExecutor executor)
{
var currentTime = SystemTime.UtcNow;
var currentExpiryThresholdTime = currentTime.AddHours(-Settings.HoursToKeepMessagesBeforeExpiring);
logger.Debug("Trying to find expired documents to delete (with threshold {0})", currentExpiryThresholdTime.ToString(Default.DateTimeFormatsToWrite, CultureInfo.InvariantCulture));
const string queryString = "Status:3 OR Status:4";
var query = new IndexQuery
{
Start = 0,
PageSize = deletionBatchSize,
Cutoff = currentTime,
Query = queryString,
FieldsToFetch = new[]
{
"__document_id",
"ProcessedAt",
"MessageMetadata"
},
SortedFields = new[]
{
new SortedField("ProcessedAt")
{
Field = "ProcessedAt",
Descending = false
}
},
};
try
{
var docsToExpire = 0;
// we may be receiving a LOT of documents to delete, so we are going to skip
// the cache for that, to avoid filling it up very quickly
var stopwatch = Stopwatch.StartNew();
int deletionCount;
using (DocumentCacher.SkipSettingDocumentsInDocumentCache())
using (Database.DisableAllTriggersForCurrentThread())
using (var cts = new CancellationTokenSource())
{
var documentWithCurrentThresholdTimeReached = false;
var items = new List<ICommandData>(deletionBatchSize);
var attachments = new List<string>(deletionBatchSize);
try
{
Database.Query(indexName, query, CancellationTokenSource.CreateLinkedTokenSource(Database.WorkContext.CancellationToken, cts.Token).Token,
null,
doc =>
{
if (documentWithCurrentThresholdTimeReached)
{
return;
}
if (doc.Value<DateTime>("ProcessedAt") >= currentExpiryThresholdTime)
{
documentWithCurrentThresholdTimeReached = true;
cts.Cancel();
return;
}
var id = doc.Value<string>("__document_id");
if (!string.IsNullOrEmpty(id))
{
items.Add(new DeleteCommandData
{
Key = id
});
var bodyNotStored = doc.SelectToken("MessageMetadata.BodyNotStored", errorWhenNoMatch: false);
if (bodyNotStored == null || bodyNotStored.Value<bool>() == false)
{
var msgId = doc.SelectToken("MessageMetadata.MessageId", errorWhenNoMatch: false);
if (msgId != null)
{
var attachmentId = "messagebodies/" + msgId.Value<string>();
attachments.Add(attachmentId);
}
}
}
});
}
catch (OperationCanceledException)
{
//Ignore
}
logger.Debug("Batching deletion of {0} documents.",items.Count);
docsToExpire += items.Count;
var results = Database.Batch(items.ToArray());
Database.TransactionalStorage.Batch(accessor =>
{
foreach (var attach in attachments)
{
accessor.Attachments.DeleteAttachment(attach, null);
}
});
deletionCount = results.Count(x => x.Deleted == true);
items.Clear();
}
if (docsToExpire == 0)
{
logger.Debug("No expired documents found");
}
else
{
logger.Debug("Deleted {0} out of {1} expired documents batch - Execution time:{2}ms", deletionCount, docsToExpire, stopwatch.ElapsedMilliseconds);
}
}
catch (Exception e)
{
logger.ErrorException("Error when trying to find expired documents", e);
}
}
