/// <summary> /// Runs the scheduler /// </summary> /// <returns>An awaitable task</returns> private async Task SchedulerRunAsync() { // Wait for the first startup if (m_isFirstActivation) { m_isFirstActivation = false; await Task.Delay(ProcessingStartupDelay); } // The running tasks var activeTasks = new List <KeyValuePair <long, Task> >(); // The last time a task was removed var removalTime = new DateTime(0); // Set up cancellation var cancelTask = new TaskCompletionSource <bool>(); m_cancelSource.Token.Register(() => cancelTask.TrySetCanceled()); while (true) { // Handle completed/failed tasks for (var i = activeTasks.Count - 1; i >= 0; i--) { var at = activeTasks[i]; if (at.Value.IsCompleted) { activeTasks.RemoveAt(i); if (removalTime.Ticks == 0) { removalTime = DateTime.Now + OldTaskLingerTime; } await this.RunInTransactionAsync(db => { var el = db.SelectItemById <QueueEntry>(at.Key); // If the request failed, try to reschedule it if (at.Value.IsCanceled || at.Value.IsFaulted) { el.Retries++; if (el.Retries > MaxRetries) { el.Status = QueueEntryStatus.Failed; } else { el.NextTry = ComputeNextTry(DateTime.Now, el.Retries); el.Status = QueueEntryStatus.Waiting; } } // All good, just mark it as done else { el.Status = QueueEntryStatus.Completed; } db.UpdateItem(el); }); } } if (removalTime.Ticks > 0 && removalTime < DateTime.Now) { removalTime = new DateTime(0); var cutoff = DateTime.Now - OldTaskLingerTime; await this.RunInTransactionAsync(db => { // Remove old tasks db.Query <QueueEntry>() .Delete() .Where(x => x.Status == QueueEntryStatus.Completed || x.Status == QueueEntryStatus.Failed) .Where(x => x.LastTried > cutoff); // Remove any run tasks no longer associated with a task db.Delete <QueueRunLog>($"{db.QuotedColumnName<QueueRunLog>(nameof(QueueRunLog.ID))} NOT IN (SELECT {db.QuotedColumnName<QueueEntry>(nameof(QueueEntry.ID))} FROM {db.QuotedTableName<QueueEntry>()})"); // Get the earliest next cleanup time var oldest = db.SelectSingle( db.Query <QueueEntry>() .Select() .Where(x => x.Status == QueueEntryStatus.Completed || x.Status == QueueEntryStatus.Failed) .OrderBy(x => x.LastTried) .Limit(1) ); if (oldest != null) { removalTime = oldest.LastTried; } }); } // If we have forced entries, run those first if (m_forcestarts.Count > 0) { List <long> ids = null; // Get the forced list, if it has any entries lock (_lock) if (m_forcestarts.Count > 0) { ids = System.Threading.Interlocked.Exchange(ref m_forcestarts, new List <long>()); } if (ids != null) { ids = ids // Make sure we do not run the tasks multiple times .Where(x => !activeTasks.Any(y => y.Key == x)) .ToList(); } if (ids.Count > 0) { var forced = await this.RunInTransactionAsync(db => db.Select( db.Query <QueueEntry>() .Select() .Where(x => x.QueueName == Name && x.Status != QueueEntryStatus.Completed ) .Where(QueryUtil.In( QueryUtil.Property( nameof(QueueEntry.ID) ), ids.Cast <object>()) ) ).ToList() ); // Start all forced tasks without obeying limits foreach (var item in forced) { activeTasks.Add( new KeyValuePair <long, Task>( item.ID, Task.Run(() => RunTaskAsync(item)) ) ); // Make sure the normal schedule also counts // the manually activated events m_ratelimiter.AddEvent(1); } } } // Get pending queue entries, ordered by NextTry var pending = await this.RunInTransactionAsync(db => db.Select( db.Query <QueueEntry>() .Select() .Where(x => x.QueueName == Name && x.Status == QueueEntryStatus.Waiting && x.NextTry <= DateTime.Now ) .OrderBy(x => x.NextTry) .Limit(activeTasks.Count - ConcurrentRequests + 1) ).ToList() ); // Keep starting tasks while (pending.Count > 0 && activeTasks.Count < ConcurrentRequests) { // If there are too many events, stop adding if (m_ratelimiter.EventCount > m_ratelimitcount) { break; } var t = pending.First(); if (t.NextTry > DateTime.Now) { break; } pending.RemoveAt(0); activeTasks.Add( new KeyValuePair <long, Task>( t.ID, Task.Run(() => RunTaskAsync(t)) ) ); m_ratelimiter.AddEvent(1); } m_activeTasks = activeTasks.Count; var delay = pending.Count == 0 ? TimeSpan.FromSeconds(30) : (DateTime.Now - pending.First().NextTry + TimeSpan.FromMilliseconds(100)); var ratelimit_delay = m_ratelimiter.WaitTime; if (ratelimit_delay.Ticks > 0) { delay = TimeSpan.FromTicks(Math.Min(delay.Ticks, ratelimit_delay.Ticks)); } if (await Task.WhenAny(m_invokeRunner.Task, Task.Delay(delay), cancelTask.Task) == m_invokeRunner.Task) { System.Threading.Interlocked.Exchange(ref m_invokeRunner, new TaskCompletionSource <bool>()); } // Stop if we are shutting down if (m_cancelSource.IsCancellationRequested) { // If we have no runners, just quit now if (activeTasks.Count == 0) { return; } // If we have runners, check on them, but do not spin await Task.Delay(200); } } }