/// <summary>
/// Renders an SQL where clause from a query element
/// </summary>
/// <param name="element">The element to use</param>
/// <returns>The SQL where clause</returns>
private string RenderWhereClause(Type type, object element, List <object> args)
{
if (element == null || element is Empty)
{
return(string.Empty);
}
if (element is And andElement)
{
return(string.Join(
" AND ",
andElement
.Items
.Select(x => RenderWhereClause(type, x, args))
.Where(x => !string.IsNullOrWhiteSpace(x))
.Select(x => $"({x})")
));
}
else if (element is Or orElement)
{
return(string.Join(
" OR ",
orElement
.Items
.Select(x => RenderWhereClause(type, x, args))
.Where(x => !string.IsNullOrWhiteSpace(x))
.Select(x => $"({x})")
));
}
else if (element is Property property)
{
return(GetTypeMap(type).QuotedColumnName(property.PropertyName));
}
else if (element is UnaryOperator unop)
{
return($"{unop.Operator} ({RenderWhereClause(type, unop.Expression, args)})");
}
else if (element is ParenthesisExpression pex)
{
return($"({RenderWhereClause(type, pex.Expression, args)})");
}
else if (element is CustomQuery cq)
{
args.AddRange(cq.Arguments ?? new object[0]);
return(cq.Value);
}
else if (element is Compare compare)
{
if (
string.Equals(compare.Operator, "IN", StringComparison.OrdinalIgnoreCase)
||
string.Equals(compare.Operator, "NOT IN", StringComparison.OrdinalIgnoreCase)
)
{
// Support for "IN" with sub-query
if (compare.RightHandSide is Query rhq)
{
if (rhq.Parsed.Type != QueryType.Select)
{
throw new ArgumentException("The query must be a select statement for exactly one column", nameof(compare.RightHandSide));
}
if (rhq.Parsed.SelectColumns.Count() != 1)
{
throw new ArgumentException("The query must be a select statement for exactly one column", nameof(compare.RightHandSide));
}
var rvp = RenderStatement(rhq);
args.AddRange(rvp.Value);
return($"{RenderWhereClause(type, compare.LeftHandSide, args)} {compare.Operator} ({rvp.Key})");
}
var rhsel = compare.RightHandSide;
IEnumerable items = null;
// Unwrap a list in parenthesis
if (rhsel is ParenthesisExpression rhspe)
{
var ve = (rhspe.Expression is Value rhspev) ? rhspev.Item : rhspe.Expression;
if (ve is IEnumerable enve)
{
items = enve;
}
else
{
var a = Array.CreateInstance(ve?.GetType() ?? typeof(object), 1);
a.SetValue(ve, 0);
items = a;
}
}
// If no parenthesis, look for a sequence inside
if (items == null && compare.RightHandSide is Value rhsv)
{
items = rhsv.Item as IEnumerable;
}
// No value, check for sequnence as a plain object
if (items == null && compare.RightHandSide is IEnumerable rhsen)
{
items = rhsen;
}
// Bounce back attempts to use a string as a char[] sequence (it implements IEnumerable)
if (items is string its)
{
items = new string[] { its }
}
;
if (items == null)
{
return(RenderWhereClause(type, QueryUtil.Equal(compare.LeftHandSide, null), args));
}
var op =
string.Equals(compare.Operator, "IN", StringComparison.OrdinalIgnoreCase)
? "="
: "!=";
// Special handling of null in lists
if (items.Cast <object>().Any(x => x == null))
{
return(RenderWhereClause(
type,
QueryUtil.Or(
QueryUtil.In(compare.LeftHandSide, items.Cast <object>().Where(x => x != null)),
QueryUtil.Compare(compare.LeftHandSide, op, null)
),
args
));
}
// No nulls, just return plain "IN" or "NOT IN"
// Does not work, it does not bind correctly to the array for some reason
// args.Add(items);
// return $"{RenderWhereClause(type, compare.LeftHandSide, args)} {compare.Operator} (?)";
// Render before, in case LHS needs to be in the args list
var lhsstr = RenderWhereClause(type, compare.LeftHandSide, args);
// Workaround is to expand to comma separated list
var qs = new List <string>();
foreach (var n in items)
{
args.Add(n);
qs.Add("?");
}
return($"{lhsstr} {compare.Operator} ({string.Join(",", qs)})");
}
// Extract the arguments, if they are arguments
var lhs = compare.LeftHandSide is Value lhsVal ? lhsVal.Item : compare.LeftHandSide;
var rhs = compare.RightHandSide is Value rhsVal ? rhsVal.Item : compare.RightHandSide;
// Special handling for enums, as they are string serialized in the database
if (IsQueryItemEnum(type, lhs) || IsQueryItemEnum(type, rhs))
{
if (!new string[] { "=", "LIKE", "!=", "NOT LIKE" }.Any(x => string.Equals(x, compare.Operator, StringComparison.InvariantCultureIgnoreCase)))
{
throw new ArgumentException("Can only compare enums with equal or not equal as they are stored as strings in the database");
}
// Force enum arguments to strings
if (lhs != null && !(lhs is QueryElement))
{
lhs = lhs.ToString();
}
if (rhs != null && !(rhs is QueryElement))
{
rhs = rhs.ToString();
}
}
// Special handling of null values to be more C# like
var anyNulls = lhs == null || rhs == null;
// Rewire gteq and lteq to handle nulls like C#
if (anyNulls && string.Equals(compare.Operator, "<="))
{
return(RenderWhereClause(type,
QueryUtil.Or(
QueryUtil.Compare(lhs, "<", rhs),
QueryUtil.Compare(lhs, "=", rhs)
)
, args));
}
if (anyNulls && string.Equals(compare.Operator, ">="))
{
return(RenderWhereClause(type,
QueryUtil.Or(
QueryUtil.Compare(lhs, ">", rhs),
QueryUtil.Compare(lhs, "=", rhs)
)
, args));
}
// Rewire compare operator to also match nulls
if (anyNulls && (string.Equals(compare.Operator, "=") || string.Equals(compare.Operator, "LIKE", StringComparison.OrdinalIgnoreCase)))
{
if (lhs == null)
{
return($"{RenderWhereClause(type, rhs, args)} IS NULL");
}
else
{
return($"{RenderWhereClause(type, lhs, args)} IS NULL");
}
}
if (anyNulls && (string.Equals(compare.Operator, "!=") || string.Equals(compare.Operator, "NOT LIKE", StringComparison.OrdinalIgnoreCase)))
{
if (lhs == null)
{
return($"{RenderWhereClause(type, rhs, args)} IS NOT NULL");
}
else
{
return($"{RenderWhereClause(type, lhs, args)} IS NOT NULL");
}
}
return($"{RenderWhereClause(type, lhs, args)} {compare.Operator} {RenderWhereClause(type, rhs, args)}");
}
else if (element is Value ve)
{
args.Add(ve.Item);
return("?");
}
else if (element is Arithmetic arithmetic)
{
return($"{RenderWhereClause(type, arithmetic.LeftHandSide, args)} {arithmetic.Operator} {RenderWhereClause(type, arithmetic.RightHandSide, args)}");
}
else if (element is QueryElement)
{
throw new Exception($"Unexpected query element: {element.GetType()}");
}
else
{
args.Add(element);
return("?");
}
}
/// <summary>
/// Runs the scheduler
/// </summary>
/// <returns>An awaitable task</returns>
private async Task SchedulerRunAsync()
{
// Wait for the first startup
if (m_isFirstActivation)
{
m_isFirstActivation = false;
await Task.Delay(ProcessingStartupDelay);
}
// The running tasks
var activeTasks = new List <KeyValuePair <long, Task> >();
// The last time a task was removed
var removalTime = new DateTime(0);
// Set up cancellation
var cancelTask = new TaskCompletionSource <bool>();
m_cancelSource.Token.Register(() => cancelTask.TrySetCanceled());
while (true)
{
// Handle completed/failed tasks
for (var i = activeTasks.Count - 1; i >= 0; i--)
{
var at = activeTasks[i];
if (at.Value.IsCompleted)
{
activeTasks.RemoveAt(i);
if (removalTime.Ticks == 0)
{
removalTime = DateTime.Now + OldTaskLingerTime;
}
await this.RunInTransactionAsync(db =>
{
var el = db.SelectItemById <QueueEntry>(at.Key);
// If the request failed, try to reschedule it
if (at.Value.IsCanceled || at.Value.IsFaulted)
{
el.Retries++;
if (el.Retries > MaxRetries)
{
el.Status = QueueEntryStatus.Failed;
}
else
{
el.NextTry = ComputeNextTry(DateTime.Now, el.Retries);
el.Status = QueueEntryStatus.Waiting;
}
}
// All good, just mark it as done
else
{
el.Status = QueueEntryStatus.Completed;
}
db.UpdateItem(el);
});
}
}
if (removalTime.Ticks > 0 && removalTime < DateTime.Now)
{
removalTime = new DateTime(0);
var cutoff = DateTime.Now - OldTaskLingerTime;
await this.RunInTransactionAsync(db => {
// Remove old tasks
db.Query <QueueEntry>()
.Delete()
.Where(x => x.Status == QueueEntryStatus.Completed || x.Status == QueueEntryStatus.Failed)
.Where(x => x.LastTried > cutoff);
// Remove any run tasks no longer associated with a task
db.Delete <QueueRunLog>($"{db.QuotedColumnName<QueueRunLog>(nameof(QueueRunLog.ID))} NOT IN (SELECT {db.QuotedColumnName<QueueEntry>(nameof(QueueEntry.ID))} FROM {db.QuotedTableName<QueueEntry>()})");
// Get the earliest next cleanup time
var oldest = db.SelectSingle(
db.Query <QueueEntry>()
.Select()
.Where(x => x.Status == QueueEntryStatus.Completed || x.Status == QueueEntryStatus.Failed)
.OrderBy(x => x.LastTried)
.Limit(1)
);
if (oldest != null)
{
removalTime = oldest.LastTried;
}
});
}
// If we have forced entries, run those first
if (m_forcestarts.Count > 0)
{
List <long> ids = null;
// Get the forced list, if it has any entries
lock (_lock)
if (m_forcestarts.Count > 0)
{
ids = System.Threading.Interlocked.Exchange(ref m_forcestarts, new List <long>());
}
if (ids != null)
{
ids = ids
// Make sure we do not run the tasks multiple times
.Where(x => !activeTasks.Any(y => y.Key == x))
.ToList();
}
if (ids.Count > 0)
{
var forced = await this.RunInTransactionAsync(db =>
db.Select(
db.Query <QueueEntry>()
.Select()
.Where(x =>
x.QueueName == Name
&&
x.Status != QueueEntryStatus.Completed
)
.Where(QueryUtil.In(
QueryUtil.Property(
nameof(QueueEntry.ID)
),
ids.Cast <object>())
)
).ToList()
);
// Start all forced tasks without obeying limits
foreach (var item in forced)
{
activeTasks.Add(
new KeyValuePair <long, Task>(
item.ID,
Task.Run(() => RunTaskAsync(item))
)
);
// Make sure the normal schedule also counts
// the manually activated events
m_ratelimiter.AddEvent(1);
}
}
}
// Get pending queue entries, ordered by NextTry
var pending = await this.RunInTransactionAsync(db =>
db.Select(
db.Query <QueueEntry>()
.Select()
.Where(x =>
x.QueueName == Name
&&
x.Status == QueueEntryStatus.Waiting
&&
x.NextTry <= DateTime.Now
)
.OrderBy(x => x.NextTry)
.Limit(activeTasks.Count - ConcurrentRequests + 1)
).ToList()
);
// Keep starting tasks
while (pending.Count > 0 && activeTasks.Count < ConcurrentRequests)
{
// If there are too many events, stop adding
if (m_ratelimiter.EventCount > m_ratelimitcount)
{
break;
}
var t = pending.First();
if (t.NextTry > DateTime.Now)
{
break;
}
pending.RemoveAt(0);
activeTasks.Add(
new KeyValuePair <long, Task>(
t.ID,
Task.Run(() => RunTaskAsync(t))
)
);
m_ratelimiter.AddEvent(1);
}
m_activeTasks = activeTasks.Count;
var delay =
pending.Count == 0
? TimeSpan.FromSeconds(30)
: (DateTime.Now - pending.First().NextTry + TimeSpan.FromMilliseconds(100));
var ratelimit_delay = m_ratelimiter.WaitTime;
if (ratelimit_delay.Ticks > 0)
{
delay = TimeSpan.FromTicks(Math.Min(delay.Ticks, ratelimit_delay.Ticks));
}
if (await Task.WhenAny(m_invokeRunner.Task, Task.Delay(delay), cancelTask.Task) == m_invokeRunner.Task)
{
System.Threading.Interlocked.Exchange(ref m_invokeRunner, new TaskCompletionSource <bool>());
}
// Stop if we are shutting down
if (m_cancelSource.IsCancellationRequested)
{
// If we have no runners, just quit now
if (activeTasks.Count == 0)
{
return;
}
// If we have runners, check on them, but do not spin
await Task.Delay(200);
}
}
}
