/// <summary>
/// The get group value factory async.
/// </summary>
/// <param name="expressions">
/// The expressions.
/// </param>
/// <returns>
/// The <see cref="Task"/>.
/// </returns>
private AsyncGroupValueFactory GetGroupValueFactory([NotNull] IEnumerable <AliasedSqlExpression> expressions)
{
var rows = Expression.Parameter(typeof(IAsyncReadOnlyCollection <Row>), "rows");
var context = Expression.Parameter(typeof(IExecutionContext));
var lambdaVisitor = new GenericVisitor
{
(ExecutionContextExpression e) => context,
(SourceFieldExpression e) => e.CreateGroupGetter(rows),
(UnaryExpression e) => e.NodeType == ExpressionType.Convert ? (e.Operand as SourceFieldExpression)?.CreateGroupGetter(rows) : null,
};
var fields = expressions.Select(
expression =>
Expression.New(
QueryPlanBuilder.KeyValuePairConstructor,
Expression.Constant(expression.Alias),
Expression.Convert(lambdaVisitor.Visit(this.data.ConvertToLinqExpression(expression.Expression)), typeof(object)))).ToArray();
var rowGetter = Expression.NewArrayInit(typeof(KeyValuePair <string, object>), fields);
// Expression.Call(TaskExpression.Task(Expression.Call(null, typeof(AsyncEnumerableExtensions))))
var result =
Expression.Lambda <AsyncGroupValueFactory>(
rowGetter.RewriteTasksToAsyncExpression(),
context,
rows);
return(result.Compile());
}
/// <summary>
/// Evaluates the <see cref="SourceBase"/>.
/// </summary>
/// <param name="expression">
/// The expression to evaluate.
/// </param>
/// <param name="sideEffects">
/// <c>true</c> if the expression has side effects, <c>false</c> otherwise.
/// </param>
/// <returns>
/// The <see cref="object"/>.
/// </returns>
private DataSource Evaluate(SourceBase expression, out bool sideEffects)
{
try
{
if (this.parsedScript.Context.NodeData.HasSideEffects(expression, variable => this.statements.HasVariableSideEffects(variable)))
{
sideEffects = true;
return(null);
}
sideEffects = false;
var linqExpression = GenericVisitor.Visit(
(ExecutionContextExpression e) => Expression.Constant(this.statements),
this.parsedScript.Context.NodeData.ConvertToDataSource(expression));
return(Expression.Lambda <Func <DataSource> >(linqExpression).Compile().Invoke());
}
catch
{
sideEffects = true;
return(null);
}
}
private static LambdaExpression ReplaceEnumerables(IConnectQlFunctions functions, string name, LambdaExpression lambda)
{
while (lambda.Parameters.Any(p => p.Type.HasInterface(typeof(IEnumerable <>)) && p.Type != typeof(string)))
{
var replacements = lambda.Parameters
.Select(p => new
{
Original = p,
ReplaceBy = p.Type.HasInterface(typeof(IEnumerable <>)) && p.Type != typeof(string) ?
Expression.Parameter(typeof(IAsyncEnumerable <>).MakeGenericType(p.Type.GenericTypeArguments[0]), p.Name) :
p,
})
.ToArray();
var enumerableParameter = replacements.First(p => p.ReplaceBy.Type.HasInterface(typeof(IAsyncEnumerable <>)));
functions.Logger?.Warning($"Parameter {enumerableParameter.Original.Name} of function {name} is an IEnumerable<T>, this reduces performance, consider using IAsyncEnumerable<T>.");
lambda = Expression.Lambda(
GenericVisitor.Visit(
(MethodCallExpression e) =>
e.Object != enumerableParameter.Original && !e.Arguments.Contains(enumerableParameter.Original)
? null
: TaskExpression.Task(
Expression.Call(
ConnectQlFunctionsExtensions.ApplyEnumerableFunction.MakeGenericMethod(enumerableParameter.Original.Type.GenericTypeArguments[0], e.Type),
enumerableParameter.ReplaceBy,
Expression.Lambda(e, enumerableParameter.Original))),
lambda.Body),
replacements.Select(r => r.ReplaceBy));
}
return(lambda);
}
public void Visit_AddSquareFunctionality_SquareCalculatedAndSet()
{
var visitor = new GenericVisitor <VisitableBase>();
Visit_1 square1 = s => s.Property1 *= s.Property1;
Visit_2 square2 = s => s.Property2 *= s.Property2;
visitor
.AddDelegate(square1).AddDelegate(square2);
var s1 = new Visitable_1 {
Property1 = 2
};
var s2 = new Visitable_2 {
Property2 = 3
};
visitor.Visit(s1);
Assert.That(s1.Property1, Is.EqualTo(4));
visitor.Visit(s2);
Assert.That(s2.Property2, Is.EqualTo(9));
}
public void DataTypeVisitor_Generic_Simple()
{
var v = new GenericVisitor();
Assert.IsNull(v.Visit(type: null));
foreach (var t in new[] {
typeof(int),
typeof(int[]),
typeof(Func <int, int>),
})
{
var d = DataType.FromType(t);
var r = v.Visit(d);
Assert.AreEqual(t, r, t.ToString());
}
var c = new MyDataType();
Assert.AreSame(c.UnderlyingType, v.Visit(c));
v.Test();
}
/// <summary>
/// Retrieves the data from the source as an <see cref="IAsyncEnumerable{T}"/>.
/// </summary>
/// <param name="context">
/// The context.
/// </param>
/// <param name="multiPartQuery">
/// The query expression.
/// </param>
/// <returns>
/// A task returning the data set.
/// </returns>
public IAsyncEnumerable <Row> GetRows(IInternalExecutionContext context, IMultiPartQuery multiPartQuery)
{
var expressions = GenericVisitor.Visit((ExecutionContextExpression e) => Expression.Constant(context), this.FilterExpression).SplitByOrExpressions().Distinct(new ExpressionComparer()).ToArray();
var orderBy = expressions.Length > 1 ? Enumerable.Empty <OrderByExpression>() : multiPartQuery.OrderByExpressions;
var result = expressions.Select(subFilter => this.GetRows(context, this.CreateJoinQuery(context, multiPartQuery.ReplaceOrderBy(orderBy), subFilter)))
.Aggregate((current, next) => current.Union(next, new RowIdComparer()));
if (expressions.Length == 1)
{
return(result);
}
context.Logger.Verbose($"Expression contains or, splitting in {expressions.Length} parts.");
return(result.OrderBy(multiPartQuery.OrderByExpressions));
}
protected internal override Node VisitUseStatement(UseStatement node)
{
node = (UseStatement)base.VisitUseStatement(node);
var context = Expression.Parameter(typeof(IExecutionContext), "context");
var getValue = Expression.Lambda <Func <IExecutionContext, object> >(
GenericVisitor.Visit(
(ExecutionContextExpression e) => context,
(CompareExpression e) => e.CreateComparer(),
this.data.ConvertToLinqExpression(node.SettingFunction)),
context).Compile();
this.data.SetQueryPlan(node, new UseDefaultQueryPlan(node.SettingFunction.Name, node.FunctionName, getValue));
return(node);
}
protected internal override Node VisitApplySource(ApplySource node)
{
node = (ApplySource)base.VisitApplySource(node);
Expression factory = Expression.Constant(null, typeof(Func <Row, DataSource>));
var left = this.data.GetFactoryExpression(node.Left);
var right = this.data.GetFactoryExpression(node.Right);
var row = Expression.Parameter(typeof(Row), "row");
var replaced = GenericVisitor.Visit(
(SourceFieldExpression e) => Expression.Call(row, NodeDataProviderDataSourceConverter.GetMethod.MakeGenericMethod(e.Type), Expression.Constant($"{e.SourceName}.{e.FieldName}")),
(UnaryExpression e) => e.NodeType == ExpressionType.Convert && e.Operand is SourceFieldExpression
? Expression.Call(row, NodeDataProviderDataSourceConverter.GetMethod.MakeGenericMethod(e.Type), Expression.Constant($"{((SourceFieldExpression)e.Operand).SourceName}.{((SourceFieldExpression)e.Operand).FieldName}"))
: null,
right);
if (!ReferenceEquals(replaced, right))
{
factory = Expression.Constant(right);
right = GenericVisitor.Visit(
(SourceFieldExpression e) => Expression.Default(e.Type),
(UnaryExpression e) => e.NodeType == ExpressionType.Convert && e.Operand is SourceFieldExpression
? Expression.Default(e.Type)
: null,
right);
}
var apply =
node.IsOuterApply
? Evaluator.CreateJoin(
typeof(OuterApply).GetTypeInfo().DeclaredConstructors.First(),
left,
right,
factory)
: Evaluator.CreateJoin(
typeof(CrossApply).GetTypeInfo().DeclaredConstructors.First(),
left,
right,
factory);
this.data.SetFactoryExpression(node, apply);
return(node);
}
/// <summary>
/// The ranges to join filter.
/// </summary>
/// <param name="filter">
/// The filter.
/// </param>
/// <returns>
/// The <see cref="Expression"/>.
/// </returns>
private static Expression RangesToJoinFilter(Expression filter)
{
return(GenericVisitor.Visit(
(CompareExpression node) =>
{
var field = node.Left as SourceFieldExpression;
var range = node.Right as RangeExpression;
if (field == null || range == null)
{
return null;
}
switch (node.CompareType)
{
case ExpressionType.Equal:
return Expression.AndAlso(
CustomExpression.MakeCompare(ExpressionType.GreaterThanOrEqual, field, Expression.Constant(range.Min, range.Type)),
CustomExpression.MakeCompare(ExpressionType.LessThanOrEqual, field, Expression.Constant(range.Max, range.Type)));
case ExpressionType.GreaterThan:
case ExpressionType.GreaterThanOrEqual:
return CustomExpression.MakeCompare(node.CompareType, field, Expression.Constant(range.Min, range.Type));
case ExpressionType.LessThan:
case ExpressionType.LessThanOrEqual:
return CustomExpression.MakeCompare(node.CompareType, field, Expression.Constant(range.Max, range.Type));
case ExpressionType.NotEqual:
return Expression.Constant(true);
default:
return null;
}
},
filter));
}
/// <summary>
/// Initializes a new instance of the <see cref="ApplyBase"/> class.
/// </summary>
/// <param name="left">
/// The left.
/// </param>
/// <param name="right">
/// The right.
/// </param>
/// <param name="rightFactory">
/// The right Factory.
/// </param>
protected ApplyBase([NotNull] DataSource left, [NotNull] DataSource right, [CanBeNull] Expression rightFactory)
: base(new HashSet <string>(left.Aliases.Concat(right.Aliases)))
{
this.left = left;
this.right = right;
this.extraFields = rightFactory?.GetDataSourceFields(this.left).ToArray() ?? new Field[0];
var context = Expression.Parameter(typeof(IExecutionContext));
var row = Expression.Parameter(typeof(Row));
this.RightFactory = rightFactory == null
? null
: Expression.Lambda <Func <IExecutionContext, Row, DataSource> >(
GenericVisitor.Visit(
(ExecutionContextExpression e) => context,
(SourceFieldExpression f) => f.CreateGetter(row),
(UnaryExpression e) => e.NodeType == ExpressionType.Convert ? (e.Operand as SourceFieldExpression)?.CreateGetter(row, e.Type) : null,
rightFactory),
context,
row).Compile();
}
/// <summary>
/// Initializes a new instance of the <see cref="DeclareVariableQueryPlan"/> class.
/// </summary>
/// <param name="name">
/// The name.
/// </param>
/// <param name="expression">
/// The expression.
/// </param>
public DeclareVariableQueryPlan(string name, Expression expression)
{
var context = Expression.Parameter(typeof(IInternalExecutionContext), "context");
expression = GenericVisitor.Visit(
(ExecutionContextExpression e) => context,
Expression.Call(context, DeclareVariableQueryPlan.SetVariable.MakeGenericMethod(expression.Type), Expression.Constant(name), expression).RewriteTasksToAsyncExpression());
if (!expression.Type.IsConstructedGenericType || expression.Type.GetGenericTypeDefinition() != typeof(Task <>))
{
var setVariable = Expression.Lambda <Action <IInternalExecutionContext> >(expression, context).Compile();
this.evaluateVariable = ctx =>
{
setVariable(ctx);
return(Task.FromResult(true));
};
}
else
{
this.evaluateVariable = Expression.Lambda <Func <IExecutionContext, Task> >(expression, context).Compile();
}
}
/// <summary>
/// Gets a factory function for the data source of this.
/// </summary>
/// <param name="source">
/// The <see cref="SourceBase"/> AST node to generate the factory for.
/// </param>
/// <returns>
/// The <see cref="Task"/>.
/// </returns>
private Func <IExecutionContext, Task <DataSource> > GetSourceFactory(SourceBase source)
{
var context = Expression.Parameter(typeof(IExecutionContext), "context");
var row = Expression.Parameter(typeof(Row), "row");
var replaceContext = new GenericVisitor
{
(ExecutionContextExpression e) => context,
(SourceFieldExpression e) => e.CreateGetter(row),
(GenericVisitor visitor, CompareExpression e) => e.CreateComparer(),
(UnaryExpression e) => e.NodeType == ExpressionType.Convert && e.Operand is SourceFieldExpression
? ((SourceFieldExpression)e.Operand).CreateGetter(row, e.Type)
: null,
};
var sourceFactoryLambda = Expression.Lambda(replaceContext.Visit(this.data.ConvertToDataSource(source)), context).RewriteTasksToAsyncExpression();
if (sourceFactoryLambda.ReturnType != typeof(Task <DataSource>))
{
sourceFactoryLambda = Expression.Lambda(Expression.Call(((Func <DataSource, Task <DataSource> >)Task.FromResult).GetMethodInfo(), sourceFactoryLambda.Body), sourceFactoryLambda.Parameters);
}
return(((Expression <Func <IExecutionContext, Task <DataSource> > >)sourceFactoryLambda).Compile());
}
/// <summary>
/// The ranges to join filter.
/// </summary>
/// <param name="filter">
/// The filter.
/// </param>
/// <returns>
/// The <see cref="Expression"/>.
/// </returns>
private static Expression RangesToJoinFilter(Expression filter)
{
return(GenericVisitor.Visit(
(CompareExpression node) =>
{
var rightRange = node.Right as RangeExpression;
if (rightRange == null)
{
var leftRange = node.Left as RangeExpression;
if (leftRange == null)
{
return null;
}
switch (node.CompareType)
{
case ExpressionType.Equal:
return Expression.AndAlso(
CustomExpression.MakeCompare(ExpressionType.GreaterThanOrEqual, leftRange.MinExpression, node.Right),
CustomExpression.MakeCompare(ExpressionType.LessThanOrEqual, leftRange.MaxExpression, node.Right));
case ExpressionType.GreaterThan:
case ExpressionType.GreaterThanOrEqual:
return CustomExpression.MakeCompare(node.CompareType, leftRange.MinExpression, node.Right);
case ExpressionType.LessThan:
case ExpressionType.LessThanOrEqual:
return CustomExpression.MakeCompare(node.CompareType, leftRange.MaxExpression, node.Right);
case ExpressionType.NotEqual:
return Expression.Constant(true);
default:
return null;
}
}
switch (node.CompareType)
{
case ExpressionType.Equal:
return Expression.AndAlso(
CustomExpression.MakeCompare(ExpressionType.GreaterThanOrEqual, node.Left, rightRange.MinExpression),
CustomExpression.MakeCompare(ExpressionType.LessThanOrEqual, node.Left, rightRange.MaxExpression));
case ExpressionType.GreaterThan:
case ExpressionType.GreaterThanOrEqual:
return CustomExpression.MakeCompare(node.CompareType, node.Left, rightRange.MinExpression);
case ExpressionType.LessThan:
case ExpressionType.LessThanOrEqual:
return CustomExpression.MakeCompare(node.CompareType, node.Left, rightRange.MaxExpression);
case ExpressionType.NotEqual:
return Expression.Constant(true);
default:
return null;
}
},
filter));
}
public static Expression GetFilter([NotNull] this IMultiPartQuery query, IExecutionContext context)
{
return(query.FilterExpression == null ? null : GenericVisitor.Visit((ExecutionContextExpression e) => Expression.Constant(context), query.FilterExpression));
}
/// <summary>
/// Gets a function that creates the values for the selected records from a row.
/// </summary>
/// <param name="expressions">
/// The <see cref="AliasedSqlExpression"/>s to get the values for.
/// </param>
/// <param name="fieldList">
/// A collection that will be filled with all the fields that are used in the select statement.
/// </param>
/// <param name="wildCardAliasList">
/// A collection that will be filled with all the wildcards that are used in the select statement.
/// </param>
/// <param name="allSourceAliases">
/// Contains all source aliases in the query, so when a global wildcard is found, all fields are added.
/// </param>
/// <returns>
/// A delegate.
/// </returns>
private Delegate GetValueFactory([NotNull] IEnumerable <AliasedSqlExpression> expressions, ICollection <IField> fieldList, ICollection <string> wildCardAliasList, IEnumerable <string> allSourceAliases)
{
var row = Expression.Parameter(typeof(Row), "row");
var context = Expression.Parameter(typeof(IExecutionContext), "context");
var fieldFactories = new List <LambdaExpression>();
var lambdaVisitor = new GenericVisitor
{
(ExecutionContextExpression e) => context,
(SourceFieldExpression e) =>
{
fieldList?.Add(new Field(e.SourceName, e.FieldName));
return(e.CreateGetter(row));
},
(UnaryExpression e) => e.NodeType == ExpressionType.Convert ? (e.Operand as SourceFieldExpression)?.CreateGetter(row, e.Type) : null,
};
Func <string, Expression, Expression <Func <IExecutionContext, Row, KeyValuePair <string, object> > > > convertToLambda = (alias, expression) =>
{
var lambda = lambdaVisitor.Visit(expression.EvaluateAsValue());
lambda = lambda.Type == typeof(object) ? lambda : Expression.Convert(lambda, typeof(object));
return(Expression.Lambda <Func <IExecutionContext, Row, KeyValuePair <string, object> > >(Expression.New(QueryPlanBuilder.KeyValuePairConstructor, Expression.Constant(alias), lambda.CatchErrors()), context, row));
};
Expression <ValueFactory> result = null;
var wildcardIndex = 0;
var hasGlobalAlias = false;
foreach (var expression in expressions)
{
if (expression.Expression is WildcardSqlExpression wildcard)
{
if (wildcard.Source != null)
{
wildCardAliasList.Add(wildcard.Source);
}
else
{
hasGlobalAlias = true;
}
var sourcePrefix = $"{wildcard.Source}.";
var index = wildcardIndex++;
var lambda = string.IsNullOrEmpty(wildcard.Source)
? (Expression <ValueFactory>)((c, r) => r.ColumnNames.Select(cn => new KeyValuePair <string, object>($"{index}!{cn}", r[cn])))
: (c, r) => r.ColumnNames.Where(cn => cn.StartsWith(sourcePrefix)).Select(cn => new KeyValuePair <string, object>($"{index}!{cn}", r[cn]));
lambda = lambda.ReplaceParameter(lambda.Parameters[0], context).ReplaceParameter(lambda.Parameters[1], row);
if (fieldFactories.Count != 0)
{
var fields = QueryPlanBuilder.ToArrayInit <ValueFactory>(fieldFactories);
result = result == null ? fields : QueryPlanBuilder.ConcatenateLambdas(result, fields);
fieldFactories.Clear();
}
result = result == null ? lambda : QueryPlanBuilder.ConcatenateLambdas(result, lambda);
}
else
{
fieldFactories.Add(convertToLambda(expression.Alias, this.data.ConvertToLinqExpression(expression.Expression)));
}
}
if (fieldFactories.Count != 0)
{
var fields = QueryPlanBuilder.ToArrayInit <ValueFactory>(fieldFactories);
result = result == null ? fields : QueryPlanBuilder.ConcatenateLambdas(result, fields);
}
if (hasGlobalAlias)
{
foreach (var alias in allSourceAliases)
{
wildCardAliasList.Add(alias);
}
}
var asyncResult = (result ?? ((c, r) => Enumerable.Empty <KeyValuePair <string, object> >())).RewriteTasksToAsyncExpression();
return(asyncResult.ReturnType == typeof(IEnumerable <KeyValuePair <string, object> >)
? ((Expression <ValueFactory>)asyncResult).Compile()
: (Delegate)((Expression <AsyncValueFactory>)asyncResult).Compile());
}
protected internal override Node VisitInsertStatement(InsertStatement node)
{
node = (InsertStatement)base.VisitInsertStatement(node);
var context = Expression.Parameter(typeof(IExecutionContext), "context");
var replaceContext = new GenericVisitor
{
(ExecutionContextExpression e) => context,
(CompareExpression e) => e.CreateComparer(),
};
var targetFactory = Expression.Lambda <Func <IExecutionContext, DataTarget> >(replaceContext.Visit(this.data.ConvertToDataTarget(node.Target)), context).Compile();
this.data.SetQueryPlan(node, new InsertQueryPlan(targetFactory, this.data.GetQueryPlan(node.Select), node.Upsert));
return(node);
}
public Expression GetFilter(IExecutionContext context)
{
return(this.FilterExpression == null ? null : GenericVisitor.Visit((ExecutionContextExpression e) => Expression.Constant(context), this.FilterExpression));
}
protected virtual JoinQuery CreateJoinQuery(IExecutionContext context, [NotNull] IMultiPartQuery query, Expression joinFilter)
{
var filterParts = (query.FilterExpression == null ? joinFilter : Expression.AndAlso(query.GetFilter(context), joinFilter)).SplitByAndExpressions();
// Get all parts of the query that contain fields from both sources.
var joinParts = filterParts.OfType <CompareExpression>()
.Where(comparison =>
{
var fields = comparison.GetFields().Select(f => f.SourceAlias).ToArray();
return(fields.Intersect(this.Left.Aliases).Any() && fields.Intersect(this.Right.Aliases).Any());
})
.ToArray();
// What's left are the filters for only one source. These we can split in a left part, a right part, and a filter part over the result.
var filter = filterParts.Except(joinParts, new ExpressionComparer()).DefaultIfEmpty().Aggregate(Expression.AndAlso);
var leftFilter = filter.RemoveAllPartsThatAreNotInSource(this.Left);
var rightFilter = filter.RemoveAllPartsThatAreNotInSource(this.Right);
var resultFilter = filter.Except(leftFilter, rightFilter);
var joinExpression = joinParts.OrderBy(p => p, new MostSpecificComparer()).First().MoveFieldsToTheLeft(this.Left);
var ascending = joinExpression.CompareType != ExpressionType.GreaterThan && joinExpression.CompareType != ExpressionType.GreaterThanOrEqual;
var leftQuery = new MultiPartQuery
{
Fields = query.Fields
.Where(f => this.Left.Aliases.Contains(f.SourceAlias))
.Concat(joinParts.SelectMany(l => l.GetDataSourceFields(this.Left)))
.Concat(resultFilter.GetDataSourceFields(this.Left))
.Distinct(),
FilterExpression = leftFilter,
OrderByExpressions = new[]
{
new OrderByExpression(joinExpression.Left, ascending)
},
};
var rightQuery = new MultiPartQuery
{
Fields = query.Fields
.Where(f => this.Right.Aliases.Contains(f.SourceAlias))
.Concat(joinParts.SelectMany(l => l.GetDataSourceFields(this.Right)))
.Concat(resultFilter.GetDataSourceFields(this.Right))
.Distinct(),
FilterExpression = rightFilter,
OrderByExpressions = new[]
{
new OrderByExpression(joinExpression.Right, ascending)
},
};
var exraJoinFilter = GenericVisitor.Visit(
(ExecutionContextExpression e) => Expression.Constant(context),
joinParts.Skip(1).DefaultIfEmpty().Aggregate <Expression>(Expression.AndAlso));
return(new JoinQuery(
leftQuery,
rightQuery,
joinExpression.Left.GetRowExpression <Row>(),
joinExpression.CompareType,
joinExpression.Right.GetRowExpression <Row>(),
joinParts.DefaultIfEmpty <Expression>().Aggregate(Expression.AndAlso),
exraJoinFilter.GetJoinFunction(this.Left),
query.OrderByExpressions,
resultFilter));
}