/// <summary>
/// Runs the scheduler
/// </summary>
/// <returns>An awaitable task</returns>
private async Task SchedulerRunAsync()
{
// Wait for the first startup
if (m_isFirstActivation)
{
m_isFirstActivation = false;
await Task.Delay(ProcessingStartupDelay);
}
// The running tasks
var activeTasks = new List <KeyValuePair <long, Task> >();
// The last time a task was removed
var removalTime = new DateTime(0);
// Set up cancellation
var cancelTask = new TaskCompletionSource <bool>();
m_cancelSource.Token.Register(() => cancelTask.TrySetCanceled());
while (true)
{
// Handle completed/failed tasks
for (var i = activeTasks.Count - 1; i >= 0; i--)
{
var at = activeTasks[i];
if (at.Value.IsCompleted)
{
activeTasks.RemoveAt(i);
if (removalTime.Ticks == 0)
{
removalTime = DateTime.Now + OldTaskLingerTime;
}
await this.RunInTransactionAsync(db =>
{
var el = db.SelectItemById <QueueEntry>(at.Key);
// If the request failed, try to reschedule it
if (at.Value.IsCanceled || at.Value.IsFaulted)
{
el.Retries++;
if (el.Retries > MaxRetries)
{
el.Status = QueueEntryStatus.Failed;
}
else
{
el.NextTry = ComputeNextTry(DateTime.Now, el.Retries);
el.Status = QueueEntryStatus.Waiting;
}
}
// All good, just mark it as done
else
{
el.Status = QueueEntryStatus.Completed;
}
db.UpdateItem(el);
});
}
}
if (removalTime.Ticks > 0 && removalTime < DateTime.Now)
{
removalTime = new DateTime(0);
var cutoff = DateTime.Now - OldTaskLingerTime;
await this.RunInTransactionAsync(db => {
// Remove old tasks
db.Query <QueueEntry>()
.Delete()
.Where(x => x.Status == QueueEntryStatus.Completed || x.Status == QueueEntryStatus.Failed)
.Where(x => x.LastTried > cutoff);
// Remove any run tasks no longer associated with a task
db.Delete <QueueRunLog>($"{db.QuotedColumnName<QueueRunLog>(nameof(QueueRunLog.ID))} NOT IN (SELECT {db.QuotedColumnName<QueueEntry>(nameof(QueueEntry.ID))} FROM {db.QuotedTableName<QueueEntry>()})");
// Get the earliest next cleanup time
var oldest = db.SelectSingle(
db.Query <QueueEntry>()
.Select()
.Where(x => x.Status == QueueEntryStatus.Completed || x.Status == QueueEntryStatus.Failed)
.OrderBy(x => x.LastTried)
.Limit(1)
);
if (oldest != null)
{
removalTime = oldest.LastTried;
}
});
}
// If we have forced entries, run those first
if (m_forcestarts.Count > 0)
{
List <long> ids = null;
// Get the forced list, if it has any entries
lock (_lock)
if (m_forcestarts.Count > 0)
{
ids = System.Threading.Interlocked.Exchange(ref m_forcestarts, new List <long>());
}
if (ids != null)
{
ids = ids
// Make sure we do not run the tasks multiple times
.Where(x => !activeTasks.Any(y => y.Key == x))
.ToList();
}
if (ids.Count > 0)
{
var forced = await this.RunInTransactionAsync(db =>
db.Select(
db.Query <QueueEntry>()
.Select()
.Where(x =>
x.QueueName == Name
&&
x.Status != QueueEntryStatus.Completed
)
.Where(QueryUtil.In(
QueryUtil.Property(
nameof(QueueEntry.ID)
),
ids.Cast <object>())
)
).ToList()
);
// Start all forced tasks without obeying limits
foreach (var item in forced)
{
activeTasks.Add(
new KeyValuePair <long, Task>(
item.ID,
Task.Run(() => RunTaskAsync(item))
)
);
// Make sure the normal schedule also counts
// the manually activated events
m_ratelimiter.AddEvent(1);
}
}
}
// Get pending queue entries, ordered by NextTry
var pending = await this.RunInTransactionAsync(db =>
db.Select(
db.Query <QueueEntry>()
.Select()
.Where(x =>
x.QueueName == Name
&&
x.Status == QueueEntryStatus.Waiting
&&
x.NextTry <= DateTime.Now
)
.OrderBy(x => x.NextTry)
.Limit(activeTasks.Count - ConcurrentRequests + 1)
).ToList()
);
// Keep starting tasks
while (pending.Count > 0 && activeTasks.Count < ConcurrentRequests)
{
// If there are too many events, stop adding
if (m_ratelimiter.EventCount > m_ratelimitcount)
{
break;
}
var t = pending.First();
if (t.NextTry > DateTime.Now)
{
break;
}
pending.RemoveAt(0);
activeTasks.Add(
new KeyValuePair <long, Task>(
t.ID,
Task.Run(() => RunTaskAsync(t))
)
);
m_ratelimiter.AddEvent(1);
}
m_activeTasks = activeTasks.Count;
var delay =
pending.Count == 0
? TimeSpan.FromSeconds(30)
: (DateTime.Now - pending.First().NextTry + TimeSpan.FromMilliseconds(100));
var ratelimit_delay = m_ratelimiter.WaitTime;
if (ratelimit_delay.Ticks > 0)
{
delay = TimeSpan.FromTicks(Math.Min(delay.Ticks, ratelimit_delay.Ticks));
}
if (await Task.WhenAny(m_invokeRunner.Task, Task.Delay(delay), cancelTask.Task) == m_invokeRunner.Task)
{
System.Threading.Interlocked.Exchange(ref m_invokeRunner, new TaskCompletionSource <bool>());
}
// Stop if we are shutting down
if (m_cancelSource.IsCancellationRequested)
{
// If we have no runners, just quit now
if (activeTasks.Count == 0)
{
return;
}
// If we have runners, check on them, but do not spin
await Task.Delay(200);
}
}
}
