protected virtual JoinQuery CreateJoinQuery(IExecutionContext context, [NotNull] IMultiPartQuery query) { var filter = query.GetFilter(context); var leftFilter = filter.RemoveAllPartsThatAreNotInSource(this.Left); var rightFilter = filter.RemoveAllPartsThatAreNotInSource(this.Right); var resultFilter = filter.SplitByAndExpressions() .Except(leftFilter.SplitByAndExpressions(), JoinSourceBase.ExpressionComparer) .Except(rightFilter.SplitByAndExpressions(), JoinSourceBase.ExpressionComparer) .DefaultIfEmpty() .Aggregate(Expression.AndAlso); var leftQuery = new MultiPartQuery { Fields = query.GetUsedFields(this.Left, resultFilter), FilterExpression = leftFilter, }; var rightQuery = new MultiPartQuery { Fields = query.GetUsedFields(this.Right, resultFilter), FilterExpression = leftFilter, }; return(new JoinQuery(leftQuery, rightQuery, resultFilter, query.OrderByExpressions)); }
/// <summary> /// Initializes a new instance of the <see cref="JoinQuery"/> class. /// </summary> /// <param name="leftQuery"> /// The left query. /// </param> /// <param name="rightQuery"> /// The right query. /// </param> /// <param name="resultFilter"> /// The filter. /// </param> /// <param name="orderBy"> /// The order by. /// </param> public JoinQuery(IMultiPartQuery leftQuery, IMultiPartQuery rightQuery, Expression resultFilter, [NotNull] IEnumerable <IOrderByExpression> orderBy) { this.LeftQuery = leftQuery; this.RightQuery = rightQuery; this.ResultFilter = resultFilter; this.OrderBy = orderBy.ToArray(); }
public static IMultiPartQuery ReplaceOrderBy([NotNull] this IMultiPartQuery expression, IEnumerable <IOrderByExpression> orderby) { return(new MultiPartQuery { Count = expression.Count, Fields = expression.Fields, OrderByExpressions = orderby, FilterExpression = expression.FilterExpression, WildcardAliases = expression.WildcardAliases, }); }
public static IMultiPartQuery ReplaceFilter([NotNull] this IMultiPartQuery expression, Expression filter) { return(new MultiPartQuery { Count = expression.Count, Fields = expression.Fields, OrderByExpressions = expression.OrderByExpressions, FilterExpression = filter, WildcardAliases = expression.WildcardAliases, }); }
/// <summary> /// Initializes a new instance of the <see cref="JoinQuery"/> class. /// </summary> /// <param name="leftQuery"> /// The left query. /// </param> /// <param name="rightQuery"> /// The right query. /// </param> /// <param name="leftKey"> /// The left Key. /// </param> /// <param name="joinType"> /// The join Type. /// </param> /// <param name="rightKey"> /// The right Key. /// </param> /// <param name="joinExpression"> /// The join Expression. /// </param> /// <param name="joinFilter"> /// The join filter. /// </param> /// <param name="orderBy"> /// The order by. /// </param> /// <param name="resultFilter"> /// The filter. /// </param> public JoinQuery(IMultiPartQuery leftQuery, IMultiPartQuery rightQuery, Func <Row, object> leftKey, ExpressionType joinType, Func <Row, object> rightKey, Expression joinExpression, Func <Row, Row, bool> joinFilter, [NotNull] IEnumerable <IOrderByExpression> orderBy, Expression resultFilter) { this.LeftQuery = leftQuery; this.RightQuery = rightQuery; this.LeftKey = leftKey; this.RightKey = rightKey; this.JoinType = joinType; this.JoinExpression = joinExpression; this.JoinFilter = joinFilter; this.OrderBy = orderBy.ToArray(); this.ResultFilter = resultFilter.GetRowFilter(); }
protected override JoinQuery CreateJoinQuery(IExecutionContext context, [NotNull] IMultiPartQuery query) { var resultFilter = query.GetFilter(context); var leftQuery = new MultiPartQuery { Fields = query.GetUsedFields(this.Left, resultFilter), }; var rightQuery = new MultiPartQuery { Fields = query.GetUsedFields(this.Right, resultFilter), }; return(new JoinQuery(leftQuery, rightQuery, resultFilter, query.OrderByExpressions)); }
/// <summary> /// Retrieves the data from the source as an <see cref="IAsyncEnumerable{T}"/>. /// </summary> /// <param name="context"> /// The context. /// </param> /// <param name="multiPartQuery"> /// The query expression. /// </param> /// <returns> /// A task returning the data set. /// </returns> public IAsyncEnumerable <Row> GetRows(IInternalExecutionContext context, IMultiPartQuery multiPartQuery) { var expressions = GenericVisitor.Visit((ExecutionContextExpression e) => Expression.Constant(context), this.FilterExpression).SplitByOrExpressions().Distinct(new ExpressionComparer()).ToArray(); var orderBy = expressions.Length > 1 ? Enumerable.Empty <OrderByExpression>() : multiPartQuery.OrderByExpressions; var result = expressions.Select(subFilter => this.GetRows(context, this.CreateJoinQuery(context, multiPartQuery.ReplaceOrderBy(orderBy), subFilter))) .Aggregate((current, next) => current.Union(next, new RowIdComparer())); if (expressions.Length == 1) { return(result); } context.Logger.Verbose($"Expression contains or, splitting in {expressions.Length} parts."); return(result.OrderBy(multiPartQuery.OrderByExpressions)); }
/// <summary> /// The get data async. /// </summary> /// <param name="context"> /// The context. /// </param> /// <param name="multiPartQuery"> /// The multi part query. /// </param> /// <returns> /// The <see cref="Task"/>. /// </returns> internal override IAsyncEnumerable <Row> GetRows(IInternalExecutionContext context, [NotNull] IMultiPartQuery multiPartQuery) { var fieldReplacer = GenericVisitor.Create((SourceFieldExpression e) => CustomExpression.MakeField(e.SourceName, e.FieldName, e.Type)); var query = multiPartQuery.WildcardAliases.Contains(this.alias) ? new Query( fieldReplacer.Visit(multiPartQuery.FilterExpression), multiPartQuery.OrderByExpressions.Select(o => new OrderByExpression(fieldReplacer.Visit(o.Expression), o.Ascending)), multiPartQuery.Count) : new Query( multiPartQuery.Fields.Where(f => f.SourceAlias == this.alias).Select(f => f.FieldName), fieldReplacer.Visit(multiPartQuery.FilterExpression), multiPartQuery.OrderByExpressions.Select(o => new OrderByExpression(fieldReplacer.Visit(o.Expression), o.Ascending)), multiPartQuery.Count); var rowBuilder = new RowBuilder(this.alias); return(context .CreateAsyncEnumerable(async() => (await this.selectPlan.ExecuteAsync(context)).QueryResults.First().Rows) .Select(rowBuilder.Attach) .Where(query.GetFilter(context)?.GetRowFilter()) .OrderBy(query.GetSortOrders(context))); }
/// <summary> /// Gets the rows for the join. /// </summary> /// <param name="context"> /// The context. /// </param> /// <param name="multiPartQuery"> /// The multi part query. /// </param> /// <returns> /// The <see cref="IAsyncEnumerable{Row}"/>. /// </returns> internal override IAsyncEnumerable <Row> GetRows(IInternalExecutionContext context, [NotNull] IMultiPartQuery multiPartQuery) { var rowBuilder = new RowBuilder(); //// Build the left part by filtering by parts that contain the fields of the left side. var leftQuery = new MultiPartQuery { Fields = multiPartQuery.Fields.Where(f => this.Left.Aliases.Contains(f.SourceAlias)), FilterExpression = multiPartQuery.FilterExpression.FilterByAliases(this.Left.Aliases), WildcardAliases = multiPartQuery.WildcardAliases.Intersect(this.Left.Aliases).ToArray(), }; //// Create the enumerable. return(context.CreateAsyncEnumerable( async() => { //// Retrieve the records from the left side. var leftData = await this.Left.GetRows(context, leftQuery).MaterializeAsync().ConfigureAwait(false); var rightQuery = new MultiPartQuery { Fields = multiPartQuery.Fields.Where(f => this.Right.Aliases.Contains(f.SourceAlias)), FilterExpression = CrossJoin.RangesToJoinFilter(await this.FindRangesAsync(context, multiPartQuery.FilterExpression, leftData)), WildcardAliases = multiPartQuery.WildcardAliases.Intersect(this.Right.Aliases), }; var rightData = await this.Right.GetRows(context, rightQuery).MaterializeAsync().ConfigureAwait(false); return leftData.CrossJoin(rightData, rowBuilder.CombineRows); }) .Where(multiPartQuery.FilterExpression.GetRowFilter()) .OrderBy(multiPartQuery.OrderByExpressions) .AfterLastElement(count => context.Logger.Verbose($"{this.GetType().Name} returned {count} records."))); }
/// <summary> /// Gets the rows for the join. /// </summary> /// <param name="context"> /// The context. /// </param> /// <param name="multiPartQuery"> /// The multi part query. /// </param> /// <returns> /// The <see cref="IAsyncEnumerable{Row}"/>. /// </returns> internal override IAsyncEnumerable <Row> GetRows(IInternalExecutionContext context, [NotNull] IMultiPartQuery multiPartQuery) { var rowBuilder = new RowBuilder(); //// Build the left part by filtering by parts that contain the fields of the left side. var leftQuery = new MultiPartQuery { Fields = multiPartQuery.Fields.Where(f => this.left.Aliases.Contains(f.SourceAlias)), WildcardAliases = multiPartQuery.WildcardAliases.Intersect(this.left.Aliases).ToArray(), }; var rightQuery = new MultiPartQuery { Fields = multiPartQuery.Fields.Where(f => this.right.Aliases.Contains(f.SourceAlias)), WildcardAliases = multiPartQuery.WildcardAliases.Intersect(this.right.Aliases), }; return(this.left.GetRows(context, leftQuery).ZipAll(this.right.GetRows(context, rightQuery), rowBuilder.CombineRows) .Where(multiPartQuery.FilterExpression.GetRowFilter()) .OrderBy(multiPartQuery.OrderByExpressions) .AfterLastElement(count => context.Logger.Verbose($"{this.GetType().Name} returned {count} records."))); }
/// <summary> /// Initializes a new instance of the <see cref="SelectQueryPlan"/> class. /// </summary> /// <param name="dataSourceFactory"> /// The data source factory. /// </param> /// <param name="query"> /// The query. /// </param> /// <param name="fieldNames"> /// The field names. /// </param> /// <param name="asyncValueFactory"> /// Lambda that creates the values for a row. /// </param> public SelectQueryPlan(Func <IExecutionContext, Task <DataSource> > dataSourceFactory, IMultiPartQuery query, IEnumerable <string> fieldNames, AsyncValueFactory asyncValueFactory) : this(dataSourceFactory, query, fieldNames) { this.asyncValueFactory = asyncValueFactory; }
/// <summary> /// Initializes a new instance of the <see cref="SelectQueryPlan"/> class. /// </summary> /// <param name="dataSourceFactory"> /// The data source factory. /// </param> /// <param name="query"> /// The query. /// </param> /// <param name="fieldNames"> /// The field names. /// </param> private SelectQueryPlan(Func <IExecutionContext, Task <DataSource> > dataSourceFactory, IMultiPartQuery query, IEnumerable <string> fieldNames) { this.dataSourceFactory = dataSourceFactory; this.query = query; this.fieldNames = fieldNames; }
public static IEnumerable <IField> GetUsedFields([NotNull] this IMultiPartQuery query, object left, object filter) { return(query.Fields.Concat(query.FilterExpression.GetFields().Concat(query.OrderByExpressions.SelectMany(o => o.Expression.GetFields()))).Distinct()); }
/// <summary> /// The get rows. /// </summary> /// <param name="context"> /// The context. /// </param> /// <param name="multiPartQuery"> /// The multi-part query. /// </param> /// <returns> /// The <see cref="IAsyncEnumerable{Row}"/>. /// </returns> internal override IAsyncEnumerable <Row> GetRows([NotNull] IInternalExecutionContext context, [NotNull] IMultiPartQuery multiPartQuery) { var functionName = context.GetDisplayName(this.dataSource); var fieldReplacer = GenericVisitor.Create((SourceFieldExpression e) => CustomExpression.MakeField(e.SourceName, e.FieldName, e.Type)); var query = multiPartQuery.WildcardAliases.Contains(this.alias) ? new Query( fieldReplacer.Visit(multiPartQuery.FilterExpression), multiPartQuery.OrderByExpressions.Select(o => new OrderByExpression(fieldReplacer.Visit(o.Expression), o.Ascending)), multiPartQuery.Count) : new Query( multiPartQuery.Fields.Where(f => f.SourceAlias == this.alias).Select(f => f.FieldName), fieldReplacer.Visit(multiPartQuery.FilterExpression), multiPartQuery.OrderByExpressions.Select(o => new OrderByExpression(fieldReplacer.Visit(o.Expression), o.Ascending)), multiPartQuery.Count); Expression unsupportedFilters = null; IOrderByExpression[] unsupportedOrderByExpressions = null; // ReSharper disable once SuspiciousTypeConversion.Global, implemented in other assemblies. var expressionSupport = this.dataSource as IDataSourceFilterSupport; if (expressionSupport != null) { var parts = query.FilterExpression.SplitByAndExpressions().Cast <CompareExpression>().ToArray(); var filters = parts.ToLookup(p => expressionSupport.SupportsExpression(p), e => (Expression)e); var supportedFilter = filters[true].DefaultIfEmpty().Aggregate(Expression.AndAlso); unsupportedFilters = filters[false].DefaultIfEmpty().Aggregate(Expression.AndAlso); if (!query.RetrieveAllFields) { query = new Query(query.Fields.Concat(unsupportedFilters.GetFields().Select(u => u.FieldName)).Distinct(), supportedFilter, query.OrderByExpressions, query.Count); } if (unsupportedFilters != null) { context.Logger.Warning($"Data source {functionName} {this.alias} has unsupported filter {unsupportedFilters}. This could impact performance."); } } // ReSharper disable once SuspiciousTypeConversion.Global, implemented in other assemblies. var orderBySupport = this.dataSource as IDataSourceOrderBySupport; if (orderBySupport != null && !orderBySupport.SupportsOrderBy(query.OrderByExpressions)) { unsupportedOrderByExpressions = query.OrderByExpressions.ToArray(); if (!query.RetrieveAllFields) { query = new Query(query.Fields.Concat(unsupportedOrderByExpressions.SelectMany(e => e.Expression.GetFields().Select(f => f.FieldName))).Distinct(), query.FilterExpression, null, query.Count); } context.Logger.Warning($"Data source {functionName} {this.alias} has unsupported ORDER BY {string.Join(", ", unsupportedOrderByExpressions.Select(u => u.Expression + " " + (u.Ascending ? "ASC" : "DESC")))}. This could impact performance."); } var sourceName = functionName + (query.FilterExpression == null ? string.Empty : $" with query '{query.FilterExpression}'"); context.Logger.Verbose($"Retrieving data from {sourceName}."); try { var result = this.dataSource.GetRows(context, new RowBuilder(this.alias), query).AfterLastElement(count => context.Logger.Verbose($"Retrieved {count} items from {sourceName}.")); if (unsupportedFilters != null) { result = result.Where(unsupportedFilters.GetRowFilter()); } if (unsupportedOrderByExpressions != null) { result.OrderBy(unsupportedOrderByExpressions); } return(result); } catch (Exception e) { context.Logger.Error($"An error occurred while querying {sourceName}: {e.Message}."); return(context.CreateEmptyAsyncEnumerable <Row>()); } }
/// <summary> /// Retrieves the data from the source as an <see cref="IAsyncEnumerable{T}"/>. /// </summary> /// <param name="context"> /// The context. /// </param> /// <param name="multiPartQuery"> /// The query expression. Can be <c>null</c>. /// </param> /// <returns> /// A task returning the data set. /// </returns> public IAsyncEnumerable <Row> GetRows(IInternalExecutionContext context, IMultiPartQuery multiPartQuery) { return(this.GetRows(context, this.CreateJoinQuery(context, multiPartQuery))); }
/// <summary> /// The get data async. /// </summary> /// <param name="context"> /// The context. /// </param> /// <param name="query"> /// The query. /// </param> /// <returns> /// The <see cref="Task"/>. /// </returns> internal abstract IAsyncEnumerable <Row> GetRows(IInternalExecutionContext context, IMultiPartQuery query);
protected virtual JoinQuery CreateJoinQuery(IExecutionContext context, [NotNull] IMultiPartQuery query, Expression joinFilter) { var filterParts = (query.FilterExpression == null ? joinFilter : Expression.AndAlso(query.GetFilter(context), joinFilter)).SplitByAndExpressions(); // Get all parts of the query that contain fields from both sources. var joinParts = filterParts.OfType <CompareExpression>() .Where(comparison => { var fields = comparison.GetFields().Select(f => f.SourceAlias).ToArray(); return(fields.Intersect(this.Left.Aliases).Any() && fields.Intersect(this.Right.Aliases).Any()); }) .ToArray(); // What's left are the filters for only one source. These we can split in a left part, a right part, and a filter part over the result. var filter = filterParts.Except(joinParts, new ExpressionComparer()).DefaultIfEmpty().Aggregate(Expression.AndAlso); var leftFilter = filter.RemoveAllPartsThatAreNotInSource(this.Left); var rightFilter = filter.RemoveAllPartsThatAreNotInSource(this.Right); var resultFilter = filter.Except(leftFilter, rightFilter); var joinExpression = joinParts.OrderBy(p => p, new MostSpecificComparer()).First().MoveFieldsToTheLeft(this.Left); var ascending = joinExpression.CompareType != ExpressionType.GreaterThan && joinExpression.CompareType != ExpressionType.GreaterThanOrEqual; var leftQuery = new MultiPartQuery { Fields = query.Fields .Where(f => this.Left.Aliases.Contains(f.SourceAlias)) .Concat(joinParts.SelectMany(l => l.GetDataSourceFields(this.Left))) .Concat(resultFilter.GetDataSourceFields(this.Left)) .Distinct(), FilterExpression = leftFilter, OrderByExpressions = new[] { new OrderByExpression(joinExpression.Left, ascending) }, }; var rightQuery = new MultiPartQuery { Fields = query.Fields .Where(f => this.Right.Aliases.Contains(f.SourceAlias)) .Concat(joinParts.SelectMany(l => l.GetDataSourceFields(this.Right))) .Concat(resultFilter.GetDataSourceFields(this.Right)) .Distinct(), FilterExpression = rightFilter, OrderByExpressions = new[] { new OrderByExpression(joinExpression.Right, ascending) }, }; var exraJoinFilter = GenericVisitor.Visit( (ExecutionContextExpression e) => Expression.Constant(context), joinParts.Skip(1).DefaultIfEmpty().Aggregate <Expression>(Expression.AndAlso)); return(new JoinQuery( leftQuery, rightQuery, joinExpression.Left.GetRowExpression <Row>(), joinExpression.CompareType, joinExpression.Right.GetRowExpression <Row>(), joinParts.DefaultIfEmpty <Expression>().Aggregate(Expression.AndAlso), exraJoinFilter.GetJoinFunction(this.Left), query.OrderByExpressions, resultFilter)); }
/// <summary> /// Gets the rows for the join. /// </summary> /// <param name="context"> /// The context. /// </param> /// <param name="multiPartQuery"> /// The multi part query. /// </param> /// <returns> /// The <see cref="IAsyncEnumerable{Row}"/>. /// </returns> internal override IAsyncEnumerable <Row> GetRows(IInternalExecutionContext context, [NotNull] IMultiPartQuery multiPartQuery) { var rowBuilder = new RowBuilder(); //// For a join, we can append the filter expression (WHERE part) with the filter for the join. var filterExpression = multiPartQuery.FilterExpression == null ? this.Filter : Expression.AndAlso(multiPartQuery.FilterExpression, this.Filter); //// Build the left part by filtering by parts that contain the fields of the left side. var leftQuery = new MultiPartQuery { Fields = multiPartQuery.Fields.Where(f => this.Left.Aliases.Contains(f.SourceAlias)).Union(this.Filter.GetFields().Where(f => this.Left.Aliases.Contains(f.SourceAlias))), FilterExpression = filterExpression.FilterByAliases(this.Left.Aliases).Simplify(context), WildcardAliases = multiPartQuery.WildcardAliases.Intersect(this.Left.Aliases).ToArray(), }; //// Create the enumerable. return(context.CreateAsyncEnumerable( async() => { //// Split by OR-expressions, var joinsParts = this.Filter.SplitByOrExpressions() //// Then, re-order AND expressions, so the most specific comparer is in front. .Select(part => part.Simplify(context).SplitByAndExpressions().Cast <CompareExpression>() .Select(c => c.MoveFieldsToTheLeft(this.Left)) .OrderBy(p => p, MostSpecificComparer.Default) .ToArray()) .ToArray(); //// Retrieve the records from the left side. var leftData = await this.Left.GetRows(context, leftQuery).MaterializeAsync().ConfigureAwait(false); if (leftData.Count == 0) { return context.CreateEmptyAsyncEnumerable <Row>(); } //// Build the right query, by using the var rightQuery = new MultiPartQuery { Fields = multiPartQuery.Fields.Where(f => this.Right.Aliases.Contains(f.SourceAlias)).Union(this.Filter.GetDataSourceFields(this.Right)), FilterExpression = JoinBase.RangesToJoinFilter(await this.FindRangesAsync(context, filterExpression, leftData)), WildcardAliases = multiPartQuery.WildcardAliases.Intersect(this.Right.Aliases), }; var rightData = await this.Right.GetRows(context, rightQuery).MaterializeAsync().ConfigureAwait(false); var result = this.CombineResults(joinsParts, rowBuilder, leftData, rightData); result = await result.MaterializeAsync(); return result; }) .Where(multiPartQuery.FilterExpression.GetRowFilter()) .OrderBy(multiPartQuery.OrderByExpressions) .AfterLastElement(count => context.Logger.Verbose($"{this.GetType().Name} returned {count} records."))); }
public static Expression GetFilter([NotNull] this IMultiPartQuery query, IExecutionContext context) { return(query.FilterExpression == null ? null : GenericVisitor.Visit((ExecutionContextExpression e) => Expression.Constant(context), query.FilterExpression)); }