/// <summary> /// Gets the rows for the join. /// </summary> /// <param name="context"> /// The context. /// </param> /// <param name="multiPartQuery"> /// The multi part query. /// </param> /// <returns> /// The <see cref="IAsyncEnumerable{Row}"/>. /// </returns> internal override IAsyncEnumerable <Row> GetRows(IInternalExecutionContext context, [NotNull] IMultiPartQuery multiPartQuery) { var rowBuilder = new RowBuilder(); //// Build the left part by filtering by parts that contain the fields of the left side. var leftQuery = new MultiPartQuery { Fields = multiPartQuery.Fields.Where(f => this.Left.Aliases.Contains(f.SourceAlias)), FilterExpression = multiPartQuery.FilterExpression.FilterByAliases(this.Left.Aliases), WildcardAliases = multiPartQuery.WildcardAliases.Intersect(this.Left.Aliases).ToArray(), }; //// Create the enumerable. return(context.CreateAsyncEnumerable( async() => { //// Retrieve the records from the left side. var leftData = await this.Left.GetRows(context, leftQuery).MaterializeAsync().ConfigureAwait(false); var rightQuery = new MultiPartQuery { Fields = multiPartQuery.Fields.Where(f => this.Right.Aliases.Contains(f.SourceAlias)), FilterExpression = CrossJoin.RangesToJoinFilter(await this.FindRangesAsync(context, multiPartQuery.FilterExpression, leftData)), WildcardAliases = multiPartQuery.WildcardAliases.Intersect(this.Right.Aliases), }; var rightData = await this.Right.GetRows(context, rightQuery).MaterializeAsync().ConfigureAwait(false); return leftData.CrossJoin(rightData, rowBuilder.CombineRows); }) .Where(multiPartQuery.FilterExpression.GetRowFilter()) .OrderBy(multiPartQuery.OrderByExpressions) .AfterLastElement(count => context.Logger.Verbose($"{this.GetType().Name} returned {count} records."))); }
public async Task <ExecuteResult> ExecuteAsync(IInternalExecutionContext context) { var dataSet = (await this.dataGenerator.ExecuteAsync(context)).QueryResults.First().Rows; var dataTarget = this.dataTargetFactory(context); return(new ExecuteResult(await dataTarget.WriteRowsAsync(context, dataSet, this.upsert), context.CreateEmptyAsyncEnumerable <Row>())); }
/// <summary> /// Initializes a new instance of the <see cref="Job"/> class. /// </summary> /// <param name="executionContext"> /// The execution context. /// </param> /// <param name="name"> /// The name. /// </param> /// <param name="plan"> /// The plan. /// </param> /// <param name="triggers"> /// The triggers. /// </param> internal Job(IInternalExecutionContext executionContext, string name, IQueryPlan plan, [NotNull] IEnumerable <IJobTrigger> triggers) { this.Name = name; this.Triggers = triggers.ToArray(); this.ExecutionContext = executionContext; this.plan = plan; }
/// <summary> /// Retrieves the data from the source as an <see cref="IAsyncEnumerable{T}"/>. /// </summary> /// <param name="context"> /// The context. /// </param> /// <param name="query"> /// The query expression. /// </param> /// <returns> /// A task returning the rows. /// </returns> protected override IAsyncEnumerable <Row> GetRows(IInternalExecutionContext context, [NotNull] JoinQuery query) { var rowBuilder = new RowBuilder(); return(this.Left.GetRows(context, query.LeftQuery) .CrossJoin(this.Right.GetRows(context, query.RightQuery), rowBuilder.CombineRows) .Where(query.ResultFilter?.GetRowFilter()) .OrderBy(query.OrderBy)); }
/// <summary> /// Gets the rows for the join. /// </summary> /// <param name="context"> /// The context. /// </param> /// <param name="multiPartQuery"> /// The multi part query. /// </param> /// <returns> /// The <see cref="IAsyncEnumerable{Row}"/>. /// </returns> internal override IAsyncEnumerable <Row> GetRows(IInternalExecutionContext context, [NotNull] IMultiPartQuery multiPartQuery) { var rowBuilder = new RowBuilder(); //// For a join, we can append the filter expression (WHERE part) with the filter for the join. var filterExpression = multiPartQuery.FilterExpression == null ? this.Filter : Expression.AndAlso(multiPartQuery.FilterExpression, this.Filter); //// Build the left part by filtering by parts that contain the fields of the left side. var leftQuery = new MultiPartQuery { Fields = multiPartQuery.Fields.Where(f => this.Left.Aliases.Contains(f.SourceAlias)).Union(this.Filter.GetFields().Where(f => this.Left.Aliases.Contains(f.SourceAlias))), FilterExpression = filterExpression.FilterByAliases(this.Left.Aliases).Simplify(context), WildcardAliases = multiPartQuery.WildcardAliases.Intersect(this.Left.Aliases).ToArray(), }; //// Create the enumerable. return(context.CreateAsyncEnumerable( async() => { //// Split by OR-expressions, var joinsParts = this.Filter.SplitByOrExpressions() //// Then, re-order AND expressions, so the most specific comparer is in front. .Select(part => part.Simplify(context).SplitByAndExpressions().Cast <CompareExpression>() .Select(c => c.MoveFieldsToTheLeft(this.Left)) .OrderBy(p => p, MostSpecificComparer.Default) .ToArray()) .ToArray(); //// Retrieve the records from the left side. var leftData = await this.Left.GetRows(context, leftQuery).MaterializeAsync().ConfigureAwait(false); if (leftData.Count == 0) { return context.CreateEmptyAsyncEnumerable <Row>(); } //// Build the right query, by using the var rightQuery = new MultiPartQuery { Fields = multiPartQuery.Fields.Where(f => this.Right.Aliases.Contains(f.SourceAlias)).Union(this.Filter.GetDataSourceFields(this.Right)), FilterExpression = JoinBase.RangesToJoinFilter(await this.FindRangesAsync(context, filterExpression, leftData)), WildcardAliases = multiPartQuery.WildcardAliases.Intersect(this.Right.Aliases), }; var rightData = await this.Right.GetRows(context, rightQuery).MaterializeAsync().ConfigureAwait(false); var result = this.CombineResults(joinsParts, rowBuilder, leftData, rightData); result = await result.MaterializeAsync(); return result; }) .Where(multiPartQuery.FilterExpression.GetRowFilter()) .OrderBy(multiPartQuery.OrderByExpressions) .AfterLastElement(count => context.Logger.Verbose($"{this.GetType().Name} returned {count} records."))); }
/// <summary> /// Retrieves the data from the source as an <see cref="IAsyncEnumerable{T}"/>. /// </summary> /// <param name="context"> /// The context. /// </param> /// <param name="query"> /// The query expression. /// </param> /// <returns> /// A task returning the rows. /// </returns> protected override IAsyncEnumerable <Row> GetRows(IInternalExecutionContext context, [NotNull] JoinQuery query) { var rowBuilder = new RowBuilder(); var leftRows = this.Left.GetRows(context, query.LeftQuery); var rightRows = this.Right.GetRows(context, query.RightQuery); return (this.isInnerJoin ? leftRows.Zip(rightRows, rowBuilder.CombineRows) : leftRows.ZipAll(rightRows, rowBuilder.CombineRows) .Where(query.ResultFilter?.GetRowFilter()) .OrderBy(query.OrderBy)); }
public async Task <ExecuteResult> ExecuteAsync(IInternalExecutionContext context) { return(new ExecuteResult(await this.SubQueries.Where(p => p != null).AggregateAsync( new List <ExecuteResult>(), async(result, plan) => { var planResult = await plan.ExecuteAsync(context); result.Add(planResult); return result; }))); }
public async Task <ExecuteResult> ExecuteAsync(IInternalExecutionContext context) { var rows = (await this.dataSourceFactory(context).ConfigureAwait(false)).GetRows(context, this.query); var rowBuilder = new RowBuilder(new FieldMapping(this.fieldNames)); rows = this.asyncValueFactory != null ? rows.Select(async row => rowBuilder.CreateRow <object>(row.UniqueId, await this.asyncValueFactory(context, row))) : rows.Select(row => rowBuilder.CreateRow(row.UniqueId, this.valueFactory(context, row))); #if DEBUG rows = await rows.MaterializeAsync(); #endif return(new ExecuteResult(0, rows)); }
/// <summary> /// Retrieves the data from the source as an <see cref="IAsyncEnumerable{T}"/>. /// </summary> /// <param name="context"> /// The context. /// </param> /// <param name="multiPartQuery"> /// The query expression. /// </param> /// <returns> /// A task returning the data set. /// </returns> public IAsyncEnumerable <Row> GetRows(IInternalExecutionContext context, IMultiPartQuery multiPartQuery) { var expressions = GenericVisitor.Visit((ExecutionContextExpression e) => Expression.Constant(context), this.FilterExpression).SplitByOrExpressions().Distinct(new ExpressionComparer()).ToArray(); var orderBy = expressions.Length > 1 ? Enumerable.Empty <OrderByExpression>() : multiPartQuery.OrderByExpressions; var result = expressions.Select(subFilter => this.GetRows(context, this.CreateJoinQuery(context, multiPartQuery.ReplaceOrderBy(orderBy), subFilter))) .Aggregate((current, next) => current.Union(next, new RowIdComparer())); if (expressions.Length == 1) { return(result); } context.Logger.Verbose($"Expression contains or, splitting in {expressions.Length} parts."); return(result.OrderBy(multiPartQuery.OrderByExpressions)); }
public async Task <ExecuteResult> ExecuteAsync(IInternalExecutionContext context) { var rowBuilder = new RowBuilder(new FieldMapping(this.fields)); var result = await this.plan.ExecuteAsync(context).ConfigureAwait(false); var id = 0; var grouped = result.QueryResults[0].Rows .GroupBy(row => this.groupFields.Select(f => row[f]).ToArray(), ArrayOfObjectComparer.Default) .Select(async group => rowBuilder.CreateRow(id++, await this.rowFactory(context, group).ConfigureAwait(false))) .Where(this.having.GetRowFilter()) .OrderBy(this.orders); #if DEBUG grouped = await context.MaterializeAsync(grouped).ConfigureAwait(false); #endif return(new ExecuteResult(0, grouped)); }
/// <summary> /// Retrieves records for the join. /// </summary> /// <param name="context"> /// The execution context. /// </param> /// <param name="query"> /// The query. /// </param> /// <returns> /// A data set containing the rows retrieved from the source. /// </returns> protected override IAsyncEnumerable <Row> GetRows(IInternalExecutionContext context, [NotNull] JoinQuery query) { var rowBuilder = new RowBuilder(); return(context.CreateAsyncEnumerable( async() => { var leftData = await this.Left.GetRows(context, query.LeftQuery).MaterializeAsync().ConfigureAwait(false); var extraFilter = (await new[] { query.JoinExpression, }.ToRangedExpressionAsync(leftData, this.Right.Aliases).ConfigureAwait(false)).Select(JoinSource.RangesToJoinFilter).First(); var rightData = await this.Right.GetRows(context, query.RightQuery.ReplaceFilter(extraFilter)).MaterializeAsync().ConfigureAwait(false); return this.IsInnerJoin ? leftData.PreSortedJoin(rightData, query.LeftKey, query.JoinType, query.RightKey, query.JoinFilter, rowBuilder.CombineRows) : leftData.PreSortedLeftJoin(rightData, query.LeftKey, query.JoinType, query.RightKey, query.JoinFilter, rowBuilder.CombineRows); }) .Where(query.ResultFilter) .OrderBy(query.OrderBy)); }
/// <summary> /// Gets the rows for the join. /// </summary> /// <param name="context"> /// The context. /// </param> /// <param name="multiPartQuery"> /// The multi part query. /// </param> /// <returns> /// The <see cref="IAsyncEnumerable{Row}"/>. /// </returns> internal override IAsyncEnumerable <Row> GetRows(IInternalExecutionContext context, [NotNull] IMultiPartQuery multiPartQuery) { var rowBuilder = new RowBuilder(); //// Build the left part by filtering by parts that contain the fields of the left side. var leftQuery = new MultiPartQuery { Fields = multiPartQuery.Fields.Where(f => this.left.Aliases.Contains(f.SourceAlias)), WildcardAliases = multiPartQuery.WildcardAliases.Intersect(this.left.Aliases).ToArray(), }; var rightQuery = new MultiPartQuery { Fields = multiPartQuery.Fields.Where(f => this.right.Aliases.Contains(f.SourceAlias)), WildcardAliases = multiPartQuery.WildcardAliases.Intersect(this.right.Aliases), }; return(this.left.GetRows(context, leftQuery).ZipAll(this.right.GetRows(context, rightQuery), rowBuilder.CombineRows) .Where(multiPartQuery.FilterExpression.GetRowFilter()) .OrderBy(multiPartQuery.OrderByExpressions) .AfterLastElement(count => context.Logger.Verbose($"{this.GetType().Name} returned {count} records."))); }
/// <summary> /// The write rows async. /// </summary> /// <param name="context"> /// The context. /// </param> /// <param name="rows"> /// The rows. /// </param> /// <param name="upsert"> /// The upsert. /// </param> /// <returns> /// The <see cref="Task"/>. /// </returns> public async Task <long> WriteRowsAsync([NotNull] IInternalExecutionContext context, IAsyncEnumerable <Row> rows, bool upsert) { var targetName = context.GetDisplayName(this.target); context.Logger.Verbose($"Writing to {targetName}..."); if (context.WriteProgressInterval != 0) { var i = 0L; rows = rows.Select( a => { i++; if (i % context.WriteProgressInterval == 0) { context.Logger.Information($"Wrote {i} items."); } return(a); }); } try { var result = await this.target.WriteRowsAsync(context, rows, upsert); context.Logger.Verbose($"Wrote {result} rows to {context.GetDisplayName(this.target)}"); return(result); } catch (Exception e) { context.Logger.Error($"An error occurred while writing to {targetName}: {e.Message}."); return(0); } }
/// <summary> /// The get data async. /// </summary> /// <param name="context"> /// The context. /// </param> /// <param name="multiPartQuery"> /// The multi part query. /// </param> /// <returns> /// The <see cref="Task"/>. /// </returns> internal override IAsyncEnumerable <Row> GetRows(IInternalExecutionContext context, [NotNull] IMultiPartQuery multiPartQuery) { var fieldReplacer = GenericVisitor.Create((SourceFieldExpression e) => CustomExpression.MakeField(e.SourceName, e.FieldName, e.Type)); var query = multiPartQuery.WildcardAliases.Contains(this.alias) ? new Query( fieldReplacer.Visit(multiPartQuery.FilterExpression), multiPartQuery.OrderByExpressions.Select(o => new OrderByExpression(fieldReplacer.Visit(o.Expression), o.Ascending)), multiPartQuery.Count) : new Query( multiPartQuery.Fields.Where(f => f.SourceAlias == this.alias).Select(f => f.FieldName), fieldReplacer.Visit(multiPartQuery.FilterExpression), multiPartQuery.OrderByExpressions.Select(o => new OrderByExpression(fieldReplacer.Visit(o.Expression), o.Ascending)), multiPartQuery.Count); var rowBuilder = new RowBuilder(this.alias); return(context .CreateAsyncEnumerable(async() => (await this.selectPlan.ExecuteAsync(context)).QueryResults.First().Rows) .Select(rowBuilder.Attach) .Where(query.GetFilter(context)?.GetRowFilter()) .OrderBy(query.GetSortOrders(context))); }
/// <summary> /// Gets the joined data. /// </summary> /// <param name="context"> /// The execution context. /// </param> /// <param name="query"> /// The query. /// </param> /// <returns> /// A data set containing the rows retrieved from the source. /// </returns> protected override IAsyncEnumerable <Row> GetRows(IInternalExecutionContext context, JoinQuery query) { throw new NotImplementedException(); }
/// <summary> /// Executes the plan. /// </summary> /// <param name="context"> /// The context. /// </param> /// <returns> /// The <see cref="ExecuteResult"/>. /// </returns> public Task <ExecuteResult> ExecuteAsync([NotNull] IInternalExecutionContext context) { context.RegisterDefault(this.setting, this.functionName, this.valueFactory(context)); return(Task.FromResult(new ExecuteResult())); }
/// <summary> /// Retrieves the data from the source as an <see cref="IAsyncEnumerable{T}"/>. /// </summary> /// <param name="context"> /// The context. /// </param> /// <param name="multiPartQuery"> /// The query expression. Can be <c>null</c>. /// </param> /// <returns> /// A task returning the data set. /// </returns> public IAsyncEnumerable <Row> GetRows(IInternalExecutionContext context, IMultiPartQuery multiPartQuery) { return(this.GetRows(context, this.CreateJoinQuery(context, multiPartQuery))); }
/// <summary> /// Retrieves the data from the source as an <see cref="IAsyncEnumerable{T}"/>. /// </summary> /// <param name="context"> /// The context. /// </param> /// <param name="query"> /// The query expression. /// </param> /// <returns> /// A task returning the rows. /// </returns> protected abstract IAsyncEnumerable <Row> GetRows(IInternalExecutionContext context, JoinQuery query);
public async Task <ExecuteResult> ExecuteAsync(IInternalExecutionContext context) { await this.evaluateVariable(context); return(new ExecuteResult()); }
/// <summary> /// The get rows. /// </summary> /// <param name="context"> /// The context. /// </param> /// <param name="multiPartQuery"> /// The multi-part query. /// </param> /// <returns> /// The <see cref="IAsyncEnumerable{Row}"/>. /// </returns> internal override IAsyncEnumerable <Row> GetRows([NotNull] IInternalExecutionContext context, [NotNull] IMultiPartQuery multiPartQuery) { var functionName = context.GetDisplayName(this.dataSource); var fieldReplacer = GenericVisitor.Create((SourceFieldExpression e) => CustomExpression.MakeField(e.SourceName, e.FieldName, e.Type)); var query = multiPartQuery.WildcardAliases.Contains(this.alias) ? new Query( fieldReplacer.Visit(multiPartQuery.FilterExpression), multiPartQuery.OrderByExpressions.Select(o => new OrderByExpression(fieldReplacer.Visit(o.Expression), o.Ascending)), multiPartQuery.Count) : new Query( multiPartQuery.Fields.Where(f => f.SourceAlias == this.alias).Select(f => f.FieldName), fieldReplacer.Visit(multiPartQuery.FilterExpression), multiPartQuery.OrderByExpressions.Select(o => new OrderByExpression(fieldReplacer.Visit(o.Expression), o.Ascending)), multiPartQuery.Count); Expression unsupportedFilters = null; IOrderByExpression[] unsupportedOrderByExpressions = null; // ReSharper disable once SuspiciousTypeConversion.Global, implemented in other assemblies. var expressionSupport = this.dataSource as IDataSourceFilterSupport; if (expressionSupport != null) { var parts = query.FilterExpression.SplitByAndExpressions().Cast <CompareExpression>().ToArray(); var filters = parts.ToLookup(p => expressionSupport.SupportsExpression(p), e => (Expression)e); var supportedFilter = filters[true].DefaultIfEmpty().Aggregate(Expression.AndAlso); unsupportedFilters = filters[false].DefaultIfEmpty().Aggregate(Expression.AndAlso); if (!query.RetrieveAllFields) { query = new Query(query.Fields.Concat(unsupportedFilters.GetFields().Select(u => u.FieldName)).Distinct(), supportedFilter, query.OrderByExpressions, query.Count); } if (unsupportedFilters != null) { context.Logger.Warning($"Data source {functionName} {this.alias} has unsupported filter {unsupportedFilters}. This could impact performance."); } } // ReSharper disable once SuspiciousTypeConversion.Global, implemented in other assemblies. var orderBySupport = this.dataSource as IDataSourceOrderBySupport; if (orderBySupport != null && !orderBySupport.SupportsOrderBy(query.OrderByExpressions)) { unsupportedOrderByExpressions = query.OrderByExpressions.ToArray(); if (!query.RetrieveAllFields) { query = new Query(query.Fields.Concat(unsupportedOrderByExpressions.SelectMany(e => e.Expression.GetFields().Select(f => f.FieldName))).Distinct(), query.FilterExpression, null, query.Count); } context.Logger.Warning($"Data source {functionName} {this.alias} has unsupported ORDER BY {string.Join(", ", unsupportedOrderByExpressions.Select(u => u.Expression + " " + (u.Ascending ? "ASC" : "DESC")))}. This could impact performance."); } var sourceName = functionName + (query.FilterExpression == null ? string.Empty : $" with query '{query.FilterExpression}'"); context.Logger.Verbose($"Retrieving data from {sourceName}."); try { var result = this.dataSource.GetRows(context, new RowBuilder(this.alias), query).AfterLastElement(count => context.Logger.Verbose($"Retrieved {count} items from {sourceName}.")); if (unsupportedFilters != null) { result = result.Where(unsupportedFilters.GetRowFilter()); } if (unsupportedOrderByExpressions != null) { result.OrderBy(unsupportedOrderByExpressions); } return(result); } catch (Exception e) { context.Logger.Error($"An error occurred while querying {sourceName}: {e.Message}."); return(context.CreateEmptyAsyncEnumerable <Row>()); } }
/// <summary> /// Combines the left and right result sets. /// </summary> /// <param name="context"> /// The context. /// </param> /// <param name="leftData"> /// The left result set. /// </param> /// <param name="rightData"> /// The right result set. /// </param> /// <param name="rightQuery"> /// The query for the right side. /// </param> /// <param name="rowBuilder"> /// The row builder. /// </param> /// <returns> /// The <see cref="IAsyncEnumerable{Row}"/>. /// </returns> protected abstract IAsyncEnumerable <Row> CombineResults(IInternalExecutionContext context, IAsyncReadOnlyCollection <Row> leftData, IAsyncReadOnlyCollection <Row> rightData, MultiPartQuery rightQuery, RowBuilder rowBuilder);
/// <summary> /// Combines the left and right result sets. /// </summary> /// <param name="context"> /// The context. /// </param> /// <param name="leftData"> /// The left result set. /// </param> /// <param name="rightData"> /// The right result set. /// </param> /// <param name="rightQuery"> /// The query for the right side. /// </param> /// <param name="rowBuilder"> /// The row builder. /// </param> /// <returns> /// The <see cref="IAsyncEnumerable{Row}"/>. /// </returns> protected override IAsyncEnumerable <Row> CombineResults(IInternalExecutionContext context, IAsyncReadOnlyCollection <Row> leftData, IAsyncReadOnlyCollection <Row> rightData, MultiPartQuery rightQuery, [NotNull] RowBuilder rowBuilder) { return(this.RightFactory != null ? leftData.CrossApply(row => this.RightFactory(context, row).GetRows(context, rightQuery), rowBuilder.CombineRows) : leftData.CrossApply(row => rightData, rowBuilder.CombineRows)); }
public async Task <ExecuteResult> ExecuteAsync(IInternalExecutionContext context) { return(new ExecuteResult(new Job(context, this.name, this.plan, await this.triggersFactory(context)))); }
/// <summary> /// The get data async. /// </summary> /// <param name="context"> /// The context. /// </param> /// <param name="query"> /// The query. /// </param> /// <returns> /// The <see cref="Task"/>. /// </returns> internal abstract IAsyncEnumerable <Row> GetRows(IInternalExecutionContext context, IMultiPartQuery query);