public static IEnumerable <SearchItem> Intersect(SearchExpressionContext c)
{
if (c.args.Length < 2)
{
c.ThrowError("Invalid arguments");
}
var setExpr = c.args[0];
if (!setExpr.types.IsIterable())
{
c.ThrowError("Primary set is not iterable", setExpr.outerText);
}
string selector = null;
if (c.args[c.args.Length - 1].types.HasFlag(SearchExpressionType.Selector))
{
selector = c.args[c.args.Length - 1].innerText.ToString();
}
var set = new HashSet <SearchItem>(new ItemComparer(selector));
foreach (var r in setExpr.Execute(c))
{
if (r == null || !set.Add(r))
{
yield return(null);
}
}
// Remove from primary set all items from subsequent sets that do not intersect
foreach (var e in c.args.Skip(1))
{
if (e != null && !e.types.HasFlag(SearchExpressionType.Selector))
{
var intersectWith = new HashSet <SearchItem>();
foreach (var r in e.Execute(c))
{
if (r == null || !intersectWith.Add(r))
{
yield return(null);
}
}
set.IntersectWith(intersectWith);
}
else
{
yield return(null);
}
}
foreach (var r in set)
{
yield return(r);
}
}
public static IEnumerable <SearchItem> Compare(SearchExpressionContext c, Func <object, object, bool> comparer)
{
if (c.args.Length < 2 || c.args.Length > 3)
{
c.ThrowError($"Invalid arguments");
}
if (c.args.Length == 3 && !IsSelectorLiteral(c.args[1]))
{
c.ThrowError($"Invalid selector");
}
var setExpr = c.args[0];
if (!setExpr.types.IsIterable())
{
c.ThrowError("Primary set is not iterable", setExpr.outerText);
}
string valueSelector = null;
var compareExprIndex = 1;
if (c.args.Length == 3)
{
valueSelector = c.args[1].innerText.ToString();
compareExprIndex++;
}
var compareExpr = c.args[compareExprIndex];
var compareValueItr = compareExpr.Execute(c).FirstOrDefault(e => e != null);
if (compareValueItr == null)
{
c.ThrowError("Invalid comparer value", compareExpr.outerText);
}
var compareValue = compareValueItr.value;
if (valueSelector == null)
{
return(setExpr.Execute(c).Where(item => item != null && comparer(item.GetValue(valueSelector), compareValue)));
}
return(TaskEvaluatorManager.EvaluateMainThread(setExpr.Execute(c), item =>
{
var value = SelectorManager.SelectValue(item, c.search, valueSelector);
if (value != null && comparer(value, compareValue))
{
return item;
}
return null;
}, 25));
}
public static IEnumerable <SearchItem> Apply(SearchExpressionContext c)
{
if (c.args.Length == 0)
{
c.ThrowError("Invalid Arguments");
}
var resultStreamExpr = c.args[0];
if (!resultStreamExpr.types.IsIterable())
{
c.ThrowError($"No iterators", resultStreamExpr.outerText);
}
var exprs = new List <SearchExpression>(c.expression.parameters.Length);
for (var i = 1; i < c.expression.parameters.Length; ++i)
{
var templateExpr = c.expression.parameters[i];
var currentStreamName = $"apply@stream#{i}";
var currentStream = EvaluatorUtils.CreateStreamExpression(resultStreamExpr, currentStreamName);
var args = new[] { currentStream }.Concat(templateExpr.parameters).ToArray();
var innerText = templateExpr.innerText;
var evaluator = templateExpr.evaluator;
if (templateExpr.types.HasFlag(SearchExpressionType.Function))
{
// Back patch arguments
var restOfArguments = templateExpr.innerText.Substring(templateExpr.evaluator.name.Length + 1);
innerText = $"{templateExpr.evaluator.name}{{{currentStreamName}, {restOfArguments}}}".GetStringView();
}
else if (templateExpr.types.HasFlag(SearchExpressionType.Text))
{
// String literal used as an evaluator name:
var filterFunction = EvaluatorManager.GetEvaluatorByNameDuringEvaluation(templateExpr.innerText.ToString(), templateExpr.innerText, c);
innerText = $"{templateExpr.innerText}{{{currentStreamName}}}".GetStringView();
evaluator = filterFunction;
}
else
{
c.ThrowError($"Bad argument type", templateExpr.outerText);
}
var patchedExpr = new SearchExpression(templateExpr.types, templateExpr.outerText, innerText, evaluator, args);
exprs.Add(patchedExpr);
resultStreamExpr = patchedExpr;
}
// Pull on the last element to start the evaluation chain
return(exprs.Last().Execute(c));
}
public static IEnumerable <SearchItem> Except(SearchExpressionContext c)
{
if (c.args.Length < 2)
{
c.ThrowError("Invalid arguments");
}
var setExpr = c.args[0];
if (!setExpr.types.IsIterable())
{
c.ThrowError("Primary set is not iterable", setExpr.outerText);
}
var exceptSet = new HashSet <SearchItem>();
foreach (var r in setExpr.Execute(c))
{
if (r == null || !exceptSet.Add(r))
{
yield return(null);
}
}
// Remove from primary set all items from subsequent sets.
foreach (var e in c.args.Skip(1))
{
if (e != null)
{
foreach (var r in e.Execute(c))
{
if (r != null)
{
exceptSet.Remove(r);
}
yield return(null);
}
}
else
{
yield return(null);
}
}
// Return excepted set.
foreach (var r in exceptSet)
{
yield return(r);
}
}
public static IEnumerable <SearchItem> Where(SearchExpressionContext c)
{
if (c.args.Length == 3)
{
var keyword = c.args[0].innerText;
var setExpr = c.args[1].Execute(c);
var whereQuery = c.args[2];
if (keyword == "none")
{
return(WhereNone(c, setExpr, whereQuery));
}
else if (keyword == "any")
{
return(WhereAny(c, setExpr, whereQuery));
}
else
{
c.ThrowError($"Unknown keyword for where evaluator: {keyword}");
return(null);
}
}
else
{
var setExpr = c.args[0];
var whereConditionExpr = c.args[1];
var queryStr = whereConditionExpr.innerText;
if (whereConditionExpr.types.HasFlag(SearchExpressionType.Selector))
{
queryStr = whereConditionExpr.outerText;
}
var setResults = setExpr.Execute(c);
return(EvaluatorManager.itemQueryEngine.WhereMainThread(c, setResults, queryStr.ToString()));
}
}
public static IEnumerable <SearchItem> Select(SearchExpressionContext c)
{
if (c.args.Length < 2)
{
c.ThrowError($"Invalid arguments");
}
// Select dataset
var dataset = c.args[0].Execute(c);
var results = dataset;
var sIt = c.args.Skip(1).GetEnumerator();
while (sIt.MoveNext())
{
var selector = sIt.Current;
if (IsSelectorLiteral(selector))
{
var selectorName = selector.innerText.ToString();
var selectorAlias = c.ResolveAlias(selector);
results = TaskEvaluatorManager.EvaluateMainThread(results, item =>
{
var selectedValue = SelectorManager.SelectValue(item, c.search, selectorName, out string suggestedSelectorName);
AddSelectedValue(item, selector.innerText.ToString(), selectorAlias ?? suggestedSelectorName, selectedValue);
return(item);
});
}
else
{
results = ProcessIterableSelector(c, results, selector);
}
}
return(results);
}
public static IEnumerable <SearchItem> Range(SearchExpressionContext c)
{
var range = new RangeDouble();
var alias = c.ResolveAlias("Range");
foreach (var sr in c.args[0].Execute(c))
{
if (GetRange(sr, ref range))
{
break;
}
else
{
yield return(null);
}
}
if (!range.valid)
{
if (c.args.Length < 2)
{
c.ThrowError("No expression to end range");
}
foreach (var sr in c.args[1].Execute(c))
{
if (GetRange(sr, ref range))
{
break;
}
else
{
yield return(null);
}
}
}
if (!range.valid)
{
c.ThrowError("Incomplete range");
}
for (double d = range.start.Value; d < range.end.Value; d += 1d)
{
yield return(EvaluatorUtils.CreateItem(d, alias));
}
}
public static IEnumerable <SearchItem> Selector(SearchExpressionContext c)
{
if (!c.expression.types.IsText())
{
c.ThrowError($"Invalid selector");
}
yield return(EvaluatorUtils.CreateItem(c.ResolveAlias("Selector"), c.expression.innerText, c.expression.innerText.ToString()));
}
public static IEnumerable <SearchItem> Text(SearchExpressionContext c)
{
if (c.args.Length == 0)
{
c.ThrowError("Text needs 1 argument");
}
return(c.args.Select(e => SearchExpression.CreateItem(e.outerText.ToString())));
}
static string ResolveVariable(string query, string token, string varName, SearchExpressionContext c)
{
var v = SelectorManager.SelectValue(c, varName, out var _);
if (v == null)
{
c.ThrowError($"Cannot resolve variable {token}{varName}");
}
return(query.Replace(token + varName, v.ToString()));
}
private static SearchExpressionEvaluator GetEvaluatorByName(string name, StringView errorView, bool duringParsing, SearchExpressionContext context = default)
{
var evaluator = FindEvaluatorByName(name);
if (!evaluator.valid)
{
if (duringParsing)
{
throw new SearchExpressionParseException(GetEvaluatorNameExceptionMessage(name), errorView.startIndex, errorView.Length);
}
else
{
context.ThrowError(GetEvaluatorNameExceptionMessage(name), errorView);
}
}
return(evaluator);
}
public static void ValidateExpressionArguments(SearchExpressionContext c, IEnumerable <Signature> signatures)
{
// First pass to get all valid argument number signatures (must do a ToList to separate the 2 passes)
// Second pass to validate the argument types. The last error is kept (lowest number of arguments if no signature matches the number of argument, wrong type if there is at least one)
var lastError = "";
var errorPosition = StringView.Null;
if (signatures.Where(s => ValidateExpressionArgumentsCount(c.expression.evaluator.name, c.args, s, (msg, errorPos) => { lastError = msg; errorPosition = errorPos; })).ToList()
.Any(s => ValidateExpressionArguments(c.expression.evaluator.name, c.args, s, (msg, errorPos) => { lastError = msg; errorPosition = errorPos; })))
{
return;
}
if (!errorPosition.valid)
{
errorPosition = c.expression.innerText;
}
c.ThrowError($"Error while evaluating arguments for {c.expression.evaluator.name}. {lastError}", errorPosition);
}
public static IEnumerable <SearchItem> If(SearchExpressionContext c)
{
if (c.args.Length < 2)
{
c.ThrowError("Not enough parameters for if");
}
bool cond = false;
foreach (var item in c.args[0].Execute(c))
{
if (item == null)
{
yield return(null);
}
else
{
cond |= EvaluatorUtils.IsTrue(item);
if (!cond)
{
break;
}
}
}
if (cond)
{
foreach (var item in c.args[1].Execute(c))
{
yield return(item);
}
}
else if (c.args.Length == 2)
{
// Nothing to do.
}
else
{
foreach (var item in c.args[2].Execute(c))
{
yield return(item);
}
}
}
public static IEnumerable <SearchItem> Random(SearchExpressionContext c)
{
if (c.args.Length == 0)
{
c.ThrowError("No arguments");
}
foreach (var e in c.args)
{
if (e != null)
{
foreach (var r in Random(c, e))
{
yield return(r);
}
}
else
{
yield return(null);
}
}
}
public static IEnumerable <SearchItem> Constant(SearchExpressionContext c)
{
if (c.expression.types.HasAny(SearchExpressionType.Function))
{
using (c.runtime.Push(c.args[0], c.args.Skip(1)))
yield return(Constant(c.runtime.current).First());
}
else if (c.expression.types.HasAny(SearchExpressionType.Number))
{
yield return(SearchExpression.CreateItem(c.expression.GetNumberValue(), c.expression.alias.ToString()));
}
else if (c.expression.types.HasAny(SearchExpressionType.Text | SearchExpressionType.Keyword))
{
yield return(SearchExpression.CreateItem(c.expression.innerText.ToString(), c.expression.alias.ToString()));
}
else if (c.expression.types.HasAny(SearchExpressionType.Boolean))
{
yield return(SearchExpression.CreateItem(c.expression.GetBooleanValue(), c.expression.alias.ToString()));
}
else
{
c.ThrowError($"Invalid constant expression");
}
}
public static IEnumerable <SearchItem> Union(SearchExpressionContext c)
{
if (c.args == null || c.args.Length == 0)
{
c.ThrowError("Nothing to merge");
}
var set = new HashSet <int>();
foreach (var e in c.args)
{
foreach (var item in e.Execute(c))
{
if (item == null)
{
yield return(null);
}
else if (set.Add(item.value.GetHashCode()))
{
yield return(item);
}
}
}
}
internal IEnumerable <SearchItem> Execute(SearchExpressionContext c, SearchExpressionExecutionFlags executionFlags)
{
if (!evaluator.valid || evaluator.execute == null)
{
c.ThrowError("Invalid expression evaluator");
}
try
{
if (!evaluator.hints.HasFlag(SearchExpressionEvaluationHints.ThreadNotSupported))
{
return(Evaluate(c, executionFlags));
}
// We cannot only return the IEnumerable of the evaluator, as the iteration itself needs to be
// done on the main thread. If we return the IEnumerable itself, we will unroll the items and call the evaluator
// in the evaluation thread.
return(TaskEvaluatorManager.EvaluateMainThreadUnroll(() => Evaluate(c, executionFlags)));
}
catch (SearchExpressionEvaluatorException ex)
{
ExceptionDispatchInfo.Capture(ex).Throw();
return(null); // To stop visual studio complaining about not all code path return a value
}
}
private IEnumerable <SearchItem> Evaluate(SearchExpressionContext c, SearchExpressionExecutionFlags flags)
{
var args = new List <SearchExpression>();
foreach (var p in parameters)
{
var evalHints = p.evaluator.hints;
if (evalHints.HasFlag(SearchExpressionEvaluationHints.AlwaysExpand) ||
(p.types.HasFlag(SearchExpressionType.Expandable) && evalHints.HasFlag(SearchExpressionEvaluationHints.ExpandSupported)))
{
foreach (var exprItem in p.Execute(c, SearchExpressionExecutionFlags.Expand))
{
if (exprItem != null)
{
if (exprItem.data is SearchExpression expr)
{
args.Add(expr);
}
else
{
c.ThrowError($"cannot expand {p}");
}
}
else
{
yield return(null);
}
}
}
else if (p.types.HasFlag(SearchExpressionType.Expandable))
{
foreach (var exprItem in p.Execute(c))
{
if (exprItem == null)
{
yield return(null);
continue;
}
if (exprItem.value != null)
{
if (Utils.TryGetNumber(exprItem.value, out var d))
{
args.Add(new SearchExpression(SearchExpressionType.Number, d.ToString().GetStringView(), Parsers.ConstantEvaluator));
}
else if (exprItem.value is string s)
{
args.Add(new SearchExpression(SearchExpressionType.Text, s.GetStringView(), Parsers.ConstantEvaluator));
}
else
{
c.ThrowError("Cannot expand parameters");
}
}
else
{
c.ThrowError("Cannot expand null value");
}
}
}
else
{
args.Add(p);
}
}
using (c.runtime.Push(this, args, flags))
{
var skipNull = c.HasFlag(SearchExpressionExecutionFlags.ThreadedEvaluation) && !c.HasFlag(SearchExpressionExecutionFlags.PassNull);
var executeContext = c.runtime.current;
var timeoutWatch = new System.Diagnostics.Stopwatch();
timeoutWatch.Start();
foreach (var r in evaluator.execute(executeContext))
{
if (r != null)
{
timeoutWatch.Restart();
yield return(r);
}
else if (!skipNull)
{
if (timeoutWatch.Elapsed.TotalSeconds > 3.0d)
{
c.ThrowError("Timeout");
}
yield return(null);
}
}
}
}
public static IEnumerable <SearchItem> Alias(SearchExpressionContext c)
{
var aliasSelector = c.args.Last();
if (c.HasFlag(SearchExpressionExecutionFlags.Expand))
{
yield return(EvaluatorUtils.CreateSearchExpressionItem(new SearchExpression(c.args[0], newAlias: aliasSelector.innerText)));
}
else
{
foreach (var r in c.args[0].Execute(c))
{
if (r == null)
{
yield return(null);
continue;
}
var hasFormatString = EvaluatorUtils.GetFormatString(aliasSelector, out var formatStr);
if (hasFormatString && aliasSelector.types.HasAny(SearchExpressionType.Text))
{
r.label = EvaluatorUtils.FormatItem(c.search, r, formatStr);
}
else if (aliasSelector.types.HasAny(SearchExpressionType.Selector))
{
r.label = SelectorManager.SelectValue(r, c.search, aliasSelector.innerText.ToString()).ToString();
yield return(r);
}
else if (aliasSelector.types.HasAny(SearchExpressionType.Iterable))
{
bool valueSelected = false;
using (c.runtime.Push(r))
{
foreach (var s in aliasSelector.Execute(c))
{
if (s != null)
{
r.label = s.value.ToString();
valueSelected = true;
break;
}
else
{
yield return(null);
}
}
}
if (!valueSelected)
{
r.label = r.label = EvaluatorUtils.FormatItem(c.search, r, aliasSelector.innerText.ToString());
}
}
else
{
c.ThrowError($"Alias selector `{aliasSelector.outerText}` not supported", aliasSelector.outerText);
}
yield return(r);
}
}
}