/// <summary> /// Renders an SQL where clause from a query element /// </summary> /// <param name="element">The element to use</param> /// <returns>The SQL where clause</returns> private string RenderWhereClause(Type type, object element, List <object> args) { if (element == null || element is Empty) { return(string.Empty); } if (element is And andElement) { return(string.Join( " AND ", andElement .Items .Select(x => RenderWhereClause(type, x, args)) .Where(x => !string.IsNullOrWhiteSpace(x)) .Select(x => $"({x})") )); } else if (element is Or orElement) { return(string.Join( " OR ", orElement .Items .Select(x => RenderWhereClause(type, x, args)) .Where(x => !string.IsNullOrWhiteSpace(x)) .Select(x => $"({x})") )); } else if (element is Property property) { return(GetTypeMap(type).QuotedColumnName(property.PropertyName)); } else if (element is UnaryOperator unop) { return($"{unop.Operator} ({RenderWhereClause(type, unop.Expression, args)})"); } else if (element is ParenthesisExpression pex) { return($"({RenderWhereClause(type, pex.Expression, args)})"); } else if (element is CustomQuery cq) { args.AddRange(cq.Arguments ?? new object[0]); return(cq.Value); } else if (element is Compare compare) { if ( string.Equals(compare.Operator, "IN", StringComparison.OrdinalIgnoreCase) || string.Equals(compare.Operator, "NOT IN", StringComparison.OrdinalIgnoreCase) ) { // Support for "IN" with sub-query if (compare.RightHandSide is Query rhq) { if (rhq.Parsed.Type != QueryType.Select) { throw new ArgumentException("The query must be a select statement for exactly one column", nameof(compare.RightHandSide)); } if (rhq.Parsed.SelectColumns.Count() != 1) { throw new ArgumentException("The query must be a select statement for exactly one column", nameof(compare.RightHandSide)); } var rvp = RenderStatement(rhq); args.AddRange(rvp.Value); return($"{RenderWhereClause(type, compare.LeftHandSide, args)} {compare.Operator} ({rvp.Key})"); } var rhsel = compare.RightHandSide; IEnumerable items = null; // Unwrap a list in parenthesis if (rhsel is ParenthesisExpression rhspe) { var ve = (rhspe.Expression is Value rhspev) ? rhspev.Item : rhspe.Expression; if (ve is IEnumerable enve) { items = enve; } else { var a = Array.CreateInstance(ve?.GetType() ?? typeof(object), 1); a.SetValue(ve, 0); items = a; } } // If no parenthesis, look for a sequence inside if (items == null && compare.RightHandSide is Value rhsv) { items = rhsv.Item as IEnumerable; } // No value, check for sequnence as a plain object if (items == null && compare.RightHandSide is IEnumerable rhsen) { items = rhsen; } // Bounce back attempts to use a string as a char[] sequence (it implements IEnumerable) if (items is string its) { items = new string[] { its } } ; if (items == null) { return(RenderWhereClause(type, QueryUtil.Equal(compare.LeftHandSide, null), args)); } var op = string.Equals(compare.Operator, "IN", StringComparison.OrdinalIgnoreCase) ? "=" : "!="; // Special handling of null in lists if (items.Cast <object>().Any(x => x == null)) { return(RenderWhereClause( type, QueryUtil.Or( QueryUtil.In(compare.LeftHandSide, items.Cast <object>().Where(x => x != null)), QueryUtil.Compare(compare.LeftHandSide, op, null) ), args )); } // No nulls, just return plain "IN" or "NOT IN" // Does not work, it does not bind correctly to the array for some reason // args.Add(items); // return $"{RenderWhereClause(type, compare.LeftHandSide, args)} {compare.Operator} (?)"; // Render before, in case LHS needs to be in the args list var lhsstr = RenderWhereClause(type, compare.LeftHandSide, args); // Workaround is to expand to comma separated list var qs = new List <string>(); foreach (var n in items) { args.Add(n); qs.Add("?"); } return($"{lhsstr} {compare.Operator} ({string.Join(",", qs)})"); } // Extract the arguments, if they are arguments var lhs = compare.LeftHandSide is Value lhsVal ? lhsVal.Item : compare.LeftHandSide; var rhs = compare.RightHandSide is Value rhsVal ? rhsVal.Item : compare.RightHandSide; // Special handling for enums, as they are string serialized in the database if (IsQueryItemEnum(type, lhs) || IsQueryItemEnum(type, rhs)) { if (!new string[] { "=", "LIKE", "!=", "NOT LIKE" }.Any(x => string.Equals(x, compare.Operator, StringComparison.InvariantCultureIgnoreCase))) { throw new ArgumentException("Can only compare enums with equal or not equal as they are stored as strings in the database"); } // Force enum arguments to strings if (lhs != null && !(lhs is QueryElement)) { lhs = lhs.ToString(); } if (rhs != null && !(rhs is QueryElement)) { rhs = rhs.ToString(); } } // Special handling of null values to be more C# like var anyNulls = lhs == null || rhs == null; // Rewire gteq and lteq to handle nulls like C# if (anyNulls && string.Equals(compare.Operator, "<=")) { return(RenderWhereClause(type, QueryUtil.Or( QueryUtil.Compare(lhs, "<", rhs), QueryUtil.Compare(lhs, "=", rhs) ) , args)); } if (anyNulls && string.Equals(compare.Operator, ">=")) { return(RenderWhereClause(type, QueryUtil.Or( QueryUtil.Compare(lhs, ">", rhs), QueryUtil.Compare(lhs, "=", rhs) ) , args)); } // Rewire compare operator to also match nulls if (anyNulls && (string.Equals(compare.Operator, "=") || string.Equals(compare.Operator, "LIKE", StringComparison.OrdinalIgnoreCase))) { if (lhs == null) { return($"{RenderWhereClause(type, rhs, args)} IS NULL"); } else { return($"{RenderWhereClause(type, lhs, args)} IS NULL"); } } if (anyNulls && (string.Equals(compare.Operator, "!=") || string.Equals(compare.Operator, "NOT LIKE", StringComparison.OrdinalIgnoreCase))) { if (lhs == null) { return($"{RenderWhereClause(type, rhs, args)} IS NOT NULL"); } else { return($"{RenderWhereClause(type, lhs, args)} IS NOT NULL"); } } return($"{RenderWhereClause(type, lhs, args)} {compare.Operator} {RenderWhereClause(type, rhs, args)}"); } else if (element is Value ve) { args.Add(ve.Item); return("?"); } else if (element is Arithmetic arithmetic) { return($"{RenderWhereClause(type, arithmetic.LeftHandSide, args)} {arithmetic.Operator} {RenderWhereClause(type, arithmetic.RightHandSide, args)}"); } else if (element is QueryElement) { throw new Exception($"Unexpected query element: {element.GetType()}"); } else { args.Add(element); return("?"); } }
/// <summary> /// Runs the scheduler /// </summary> /// <returns>An awaitable task</returns> private async Task SchedulerRunAsync() { // Wait for the first startup if (m_isFirstActivation) { m_isFirstActivation = false; await Task.Delay(ProcessingStartupDelay); } // The running tasks var activeTasks = new List <KeyValuePair <long, Task> >(); // The last time a task was removed var removalTime = new DateTime(0); // Set up cancellation var cancelTask = new TaskCompletionSource <bool>(); m_cancelSource.Token.Register(() => cancelTask.TrySetCanceled()); while (true) { // Handle completed/failed tasks for (var i = activeTasks.Count - 1; i >= 0; i--) { var at = activeTasks[i]; if (at.Value.IsCompleted) { activeTasks.RemoveAt(i); if (removalTime.Ticks == 0) { removalTime = DateTime.Now + OldTaskLingerTime; } await this.RunInTransactionAsync(db => { var el = db.SelectItemById <QueueEntry>(at.Key); // If the request failed, try to reschedule it if (at.Value.IsCanceled || at.Value.IsFaulted) { el.Retries++; if (el.Retries > MaxRetries) { el.Status = QueueEntryStatus.Failed; } else { el.NextTry = ComputeNextTry(DateTime.Now, el.Retries); el.Status = QueueEntryStatus.Waiting; } } // All good, just mark it as done else { el.Status = QueueEntryStatus.Completed; } db.UpdateItem(el); }); } } if (removalTime.Ticks > 0 && removalTime < DateTime.Now) { removalTime = new DateTime(0); var cutoff = DateTime.Now - OldTaskLingerTime; await this.RunInTransactionAsync(db => { // Remove old tasks db.Query <QueueEntry>() .Delete() .Where(x => x.Status == QueueEntryStatus.Completed || x.Status == QueueEntryStatus.Failed) .Where(x => x.LastTried > cutoff); // Remove any run tasks no longer associated with a task db.Delete <QueueRunLog>($"{db.QuotedColumnName<QueueRunLog>(nameof(QueueRunLog.ID))} NOT IN (SELECT {db.QuotedColumnName<QueueEntry>(nameof(QueueEntry.ID))} FROM {db.QuotedTableName<QueueEntry>()})"); // Get the earliest next cleanup time var oldest = db.SelectSingle( db.Query <QueueEntry>() .Select() .Where(x => x.Status == QueueEntryStatus.Completed || x.Status == QueueEntryStatus.Failed) .OrderBy(x => x.LastTried) .Limit(1) ); if (oldest != null) { removalTime = oldest.LastTried; } }); } // If we have forced entries, run those first if (m_forcestarts.Count > 0) { List <long> ids = null; // Get the forced list, if it has any entries lock (_lock) if (m_forcestarts.Count > 0) { ids = System.Threading.Interlocked.Exchange(ref m_forcestarts, new List <long>()); } if (ids != null) { ids = ids // Make sure we do not run the tasks multiple times .Where(x => !activeTasks.Any(y => y.Key == x)) .ToList(); } if (ids.Count > 0) { var forced = await this.RunInTransactionAsync(db => db.Select( db.Query <QueueEntry>() .Select() .Where(x => x.QueueName == Name && x.Status != QueueEntryStatus.Completed ) .Where(QueryUtil.In( QueryUtil.Property( nameof(QueueEntry.ID) ), ids.Cast <object>()) ) ).ToList() ); // Start all forced tasks without obeying limits foreach (var item in forced) { activeTasks.Add( new KeyValuePair <long, Task>( item.ID, Task.Run(() => RunTaskAsync(item)) ) ); // Make sure the normal schedule also counts // the manually activated events m_ratelimiter.AddEvent(1); } } } // Get pending queue entries, ordered by NextTry var pending = await this.RunInTransactionAsync(db => db.Select( db.Query <QueueEntry>() .Select() .Where(x => x.QueueName == Name && x.Status == QueueEntryStatus.Waiting && x.NextTry <= DateTime.Now ) .OrderBy(x => x.NextTry) .Limit(activeTasks.Count - ConcurrentRequests + 1) ).ToList() ); // Keep starting tasks while (pending.Count > 0 && activeTasks.Count < ConcurrentRequests) { // If there are too many events, stop adding if (m_ratelimiter.EventCount > m_ratelimitcount) { break; } var t = pending.First(); if (t.NextTry > DateTime.Now) { break; } pending.RemoveAt(0); activeTasks.Add( new KeyValuePair <long, Task>( t.ID, Task.Run(() => RunTaskAsync(t)) ) ); m_ratelimiter.AddEvent(1); } m_activeTasks = activeTasks.Count; var delay = pending.Count == 0 ? TimeSpan.FromSeconds(30) : (DateTime.Now - pending.First().NextTry + TimeSpan.FromMilliseconds(100)); var ratelimit_delay = m_ratelimiter.WaitTime; if (ratelimit_delay.Ticks > 0) { delay = TimeSpan.FromTicks(Math.Min(delay.Ticks, ratelimit_delay.Ticks)); } if (await Task.WhenAny(m_invokeRunner.Task, Task.Delay(delay), cancelTask.Task) == m_invokeRunner.Task) { System.Threading.Interlocked.Exchange(ref m_invokeRunner, new TaskCompletionSource <bool>()); } // Stop if we are shutting down if (m_cancelSource.IsCancellationRequested) { // If we have no runners, just quit now if (activeTasks.Count == 0) { return; } // If we have runners, check on them, but do not spin await Task.Delay(200); } } }