/// <summary>
/// Gets the rows for the join.
/// </summary>
/// <param name="context">
/// The context.
/// </param>
/// <param name="multiPartQuery">
/// The multi part query.
/// </param>
/// <returns>
/// The <see cref="IAsyncEnumerable{Row}"/>.
/// </returns>
internal override IAsyncEnumerable <Row> GetRows(IInternalExecutionContext context, [NotNull] IMultiPartQuery multiPartQuery)
{
var rowBuilder = new RowBuilder();
//// Build the left part by filtering by parts that contain the fields of the left side.
var leftQuery = new MultiPartQuery
{
Fields = multiPartQuery.Fields.Where(f => this.Left.Aliases.Contains(f.SourceAlias)),
FilterExpression = multiPartQuery.FilterExpression.FilterByAliases(this.Left.Aliases),
WildcardAliases = multiPartQuery.WildcardAliases.Intersect(this.Left.Aliases).ToArray(),
};
//// Create the enumerable.
return(context.CreateAsyncEnumerable(
async() =>
{
//// Retrieve the records from the left side.
var leftData = await this.Left.GetRows(context, leftQuery).MaterializeAsync().ConfigureAwait(false);
var rightQuery = new MultiPartQuery
{
Fields = multiPartQuery.Fields.Where(f => this.Right.Aliases.Contains(f.SourceAlias)),
FilterExpression = CrossJoin.RangesToJoinFilter(await this.FindRangesAsync(context, multiPartQuery.FilterExpression, leftData)),
WildcardAliases = multiPartQuery.WildcardAliases.Intersect(this.Right.Aliases),
};
var rightData = await this.Right.GetRows(context, rightQuery).MaterializeAsync().ConfigureAwait(false);
return leftData.CrossJoin(rightData, rowBuilder.CombineRows);
})
.Where(multiPartQuery.FilterExpression.GetRowFilter())
.OrderBy(multiPartQuery.OrderByExpressions)
.AfterLastElement(count => context.Logger.Verbose($"{this.GetType().Name} returned {count} records.")));
}
public async Task <ExecuteResult> ExecuteAsync(IInternalExecutionContext context)
{
var dataSet = (await this.dataGenerator.ExecuteAsync(context)).QueryResults.First().Rows;
var dataTarget = this.dataTargetFactory(context);
return(new ExecuteResult(await dataTarget.WriteRowsAsync(context, dataSet, this.upsert), context.CreateEmptyAsyncEnumerable <Row>()));
}
/// <summary>
/// Initializes a new instance of the <see cref="Job"/> class.
/// </summary>
/// <param name="executionContext">
/// The execution context.
/// </param>
/// <param name="name">
/// The name.
/// </param>
/// <param name="plan">
/// The plan.
/// </param>
/// <param name="triggers">
/// The triggers.
/// </param>
internal Job(IInternalExecutionContext executionContext, string name, IQueryPlan plan, [NotNull] IEnumerable <IJobTrigger> triggers)
{
this.Name = name;
this.Triggers = triggers.ToArray();
this.ExecutionContext = executionContext;
this.plan = plan;
}
/// <summary>
/// Retrieves the data from the source as an <see cref="IAsyncEnumerable{T}"/>.
/// </summary>
/// <param name="context">
/// The context.
/// </param>
/// <param name="query">
/// The query expression.
/// </param>
/// <returns>
/// A task returning the rows.
/// </returns>
protected override IAsyncEnumerable <Row> GetRows(IInternalExecutionContext context, [NotNull] JoinQuery query)
{
var rowBuilder = new RowBuilder();
return(this.Left.GetRows(context, query.LeftQuery)
.CrossJoin(this.Right.GetRows(context, query.RightQuery), rowBuilder.CombineRows)
.Where(query.ResultFilter?.GetRowFilter())
.OrderBy(query.OrderBy));
}
/// <summary>
/// Gets the rows for the join.
/// </summary>
/// <param name="context">
/// The context.
/// </param>
/// <param name="multiPartQuery">
/// The multi part query.
/// </param>
/// <returns>
/// The <see cref="IAsyncEnumerable{Row}"/>.
/// </returns>
internal override IAsyncEnumerable <Row> GetRows(IInternalExecutionContext context, [NotNull] IMultiPartQuery multiPartQuery)
{
var rowBuilder = new RowBuilder();
//// For a join, we can append the filter expression (WHERE part) with the filter for the join.
var filterExpression = multiPartQuery.FilterExpression == null ? this.Filter : Expression.AndAlso(multiPartQuery.FilterExpression, this.Filter);
//// Build the left part by filtering by parts that contain the fields of the left side.
var leftQuery = new MultiPartQuery
{
Fields = multiPartQuery.Fields.Where(f => this.Left.Aliases.Contains(f.SourceAlias)).Union(this.Filter.GetFields().Where(f => this.Left.Aliases.Contains(f.SourceAlias))),
FilterExpression = filterExpression.FilterByAliases(this.Left.Aliases).Simplify(context),
WildcardAliases = multiPartQuery.WildcardAliases.Intersect(this.Left.Aliases).ToArray(),
};
//// Create the enumerable.
return(context.CreateAsyncEnumerable(
async() =>
{
//// Split by OR-expressions,
var joinsParts = this.Filter.SplitByOrExpressions()
//// Then, re-order AND expressions, so the most specific comparer is in front.
.Select(part => part.Simplify(context).SplitByAndExpressions().Cast <CompareExpression>()
.Select(c => c.MoveFieldsToTheLeft(this.Left))
.OrderBy(p => p, MostSpecificComparer.Default)
.ToArray())
.ToArray();
//// Retrieve the records from the left side.
var leftData = await this.Left.GetRows(context, leftQuery).MaterializeAsync().ConfigureAwait(false);
if (leftData.Count == 0)
{
return context.CreateEmptyAsyncEnumerable <Row>();
}
//// Build the right query, by using the
var rightQuery = new MultiPartQuery
{
Fields = multiPartQuery.Fields.Where(f => this.Right.Aliases.Contains(f.SourceAlias)).Union(this.Filter.GetDataSourceFields(this.Right)),
FilterExpression = JoinBase.RangesToJoinFilter(await this.FindRangesAsync(context, filterExpression, leftData)),
WildcardAliases = multiPartQuery.WildcardAliases.Intersect(this.Right.Aliases),
};
var rightData = await this.Right.GetRows(context, rightQuery).MaterializeAsync().ConfigureAwait(false);
var result = this.CombineResults(joinsParts, rowBuilder, leftData, rightData);
result = await result.MaterializeAsync();
return result;
})
.Where(multiPartQuery.FilterExpression.GetRowFilter())
.OrderBy(multiPartQuery.OrderByExpressions)
.AfterLastElement(count => context.Logger.Verbose($"{this.GetType().Name} returned {count} records.")));
}
/// <summary>
/// Retrieves the data from the source as an <see cref="IAsyncEnumerable{T}"/>.
/// </summary>
/// <param name="context">
/// The context.
/// </param>
/// <param name="query">
/// The query expression.
/// </param>
/// <returns>
/// A task returning the rows.
/// </returns>
protected override IAsyncEnumerable <Row> GetRows(IInternalExecutionContext context, [NotNull] JoinQuery query)
{
var rowBuilder = new RowBuilder();
var leftRows = this.Left.GetRows(context, query.LeftQuery);
var rightRows = this.Right.GetRows(context, query.RightQuery);
return
(this.isInnerJoin
? leftRows.Zip(rightRows, rowBuilder.CombineRows)
: leftRows.ZipAll(rightRows, rowBuilder.CombineRows)
.Where(query.ResultFilter?.GetRowFilter())
.OrderBy(query.OrderBy));
}
public async Task <ExecuteResult> ExecuteAsync(IInternalExecutionContext context)
{
return(new ExecuteResult(await this.SubQueries.Where(p => p != null).AggregateAsync(
new List <ExecuteResult>(),
async(result, plan) =>
{
var planResult = await plan.ExecuteAsync(context);
result.Add(planResult);
return result;
})));
}
public async Task <ExecuteResult> ExecuteAsync(IInternalExecutionContext context)
{
var rows = (await this.dataSourceFactory(context).ConfigureAwait(false)).GetRows(context, this.query);
var rowBuilder = new RowBuilder(new FieldMapping(this.fieldNames));
rows = this.asyncValueFactory != null
? rows.Select(async row => rowBuilder.CreateRow <object>(row.UniqueId, await this.asyncValueFactory(context, row)))
: rows.Select(row => rowBuilder.CreateRow(row.UniqueId, this.valueFactory(context, row)));
#if DEBUG
rows = await rows.MaterializeAsync();
#endif
return(new ExecuteResult(0, rows));
}
/// <summary>
/// Retrieves the data from the source as an <see cref="IAsyncEnumerable{T}"/>.
/// </summary>
/// <param name="context">
/// The context.
/// </param>
/// <param name="multiPartQuery">
/// The query expression.
/// </param>
/// <returns>
/// A task returning the data set.
/// </returns>
public IAsyncEnumerable <Row> GetRows(IInternalExecutionContext context, IMultiPartQuery multiPartQuery)
{
var expressions = GenericVisitor.Visit((ExecutionContextExpression e) => Expression.Constant(context), this.FilterExpression).SplitByOrExpressions().Distinct(new ExpressionComparer()).ToArray();
var orderBy = expressions.Length > 1 ? Enumerable.Empty <OrderByExpression>() : multiPartQuery.OrderByExpressions;
var result = expressions.Select(subFilter => this.GetRows(context, this.CreateJoinQuery(context, multiPartQuery.ReplaceOrderBy(orderBy), subFilter)))
.Aggregate((current, next) => current.Union(next, new RowIdComparer()));
if (expressions.Length == 1)
{
return(result);
}
context.Logger.Verbose($"Expression contains or, splitting in {expressions.Length} parts.");
return(result.OrderBy(multiPartQuery.OrderByExpressions));
}
public async Task <ExecuteResult> ExecuteAsync(IInternalExecutionContext context)
{
var rowBuilder = new RowBuilder(new FieldMapping(this.fields));
var result = await this.plan.ExecuteAsync(context).ConfigureAwait(false);
var id = 0;
var grouped = result.QueryResults[0].Rows
.GroupBy(row => this.groupFields.Select(f => row[f]).ToArray(), ArrayOfObjectComparer.Default)
.Select(async group => rowBuilder.CreateRow(id++, await this.rowFactory(context, group).ConfigureAwait(false)))
.Where(this.having.GetRowFilter())
.OrderBy(this.orders);
#if DEBUG
grouped = await context.MaterializeAsync(grouped).ConfigureAwait(false);
#endif
return(new ExecuteResult(0, grouped));
}
/// <summary>
/// Retrieves records for the join.
/// </summary>
/// <param name="context">
/// The execution context.
/// </param>
/// <param name="query">
/// The query.
/// </param>
/// <returns>
/// A data set containing the rows retrieved from the source.
/// </returns>
protected override IAsyncEnumerable <Row> GetRows(IInternalExecutionContext context, [NotNull] JoinQuery query)
{
var rowBuilder = new RowBuilder();
return(context.CreateAsyncEnumerable(
async() =>
{
var leftData = await this.Left.GetRows(context, query.LeftQuery).MaterializeAsync().ConfigureAwait(false);
var extraFilter = (await new[]
{
query.JoinExpression,
}.ToRangedExpressionAsync(leftData, this.Right.Aliases).ConfigureAwait(false)).Select(JoinSource.RangesToJoinFilter).First();
var rightData = await this.Right.GetRows(context, query.RightQuery.ReplaceFilter(extraFilter)).MaterializeAsync().ConfigureAwait(false);
return this.IsInnerJoin
? leftData.PreSortedJoin(rightData, query.LeftKey, query.JoinType, query.RightKey, query.JoinFilter, rowBuilder.CombineRows)
: leftData.PreSortedLeftJoin(rightData, query.LeftKey, query.JoinType, query.RightKey, query.JoinFilter, rowBuilder.CombineRows);
})
.Where(query.ResultFilter)
.OrderBy(query.OrderBy));
}
/// <summary>
/// Gets the rows for the join.
/// </summary>
/// <param name="context">
/// The context.
/// </param>
/// <param name="multiPartQuery">
/// The multi part query.
/// </param>
/// <returns>
/// The <see cref="IAsyncEnumerable{Row}"/>.
/// </returns>
internal override IAsyncEnumerable <Row> GetRows(IInternalExecutionContext context, [NotNull] IMultiPartQuery multiPartQuery)
{
var rowBuilder = new RowBuilder();
//// Build the left part by filtering by parts that contain the fields of the left side.
var leftQuery = new MultiPartQuery
{
Fields = multiPartQuery.Fields.Where(f => this.left.Aliases.Contains(f.SourceAlias)),
WildcardAliases = multiPartQuery.WildcardAliases.Intersect(this.left.Aliases).ToArray(),
};
var rightQuery = new MultiPartQuery
{
Fields = multiPartQuery.Fields.Where(f => this.right.Aliases.Contains(f.SourceAlias)),
WildcardAliases = multiPartQuery.WildcardAliases.Intersect(this.right.Aliases),
};
return(this.left.GetRows(context, leftQuery).ZipAll(this.right.GetRows(context, rightQuery), rowBuilder.CombineRows)
.Where(multiPartQuery.FilterExpression.GetRowFilter())
.OrderBy(multiPartQuery.OrderByExpressions)
.AfterLastElement(count => context.Logger.Verbose($"{this.GetType().Name} returned {count} records.")));
}
/// <summary>
/// The write rows async.
/// </summary>
/// <param name="context">
/// The context.
/// </param>
/// <param name="rows">
/// The rows.
/// </param>
/// <param name="upsert">
/// The upsert.
/// </param>
/// <returns>
/// The <see cref="Task"/>.
/// </returns>
public async Task <long> WriteRowsAsync([NotNull] IInternalExecutionContext context, IAsyncEnumerable <Row> rows, bool upsert)
{
var targetName = context.GetDisplayName(this.target);
context.Logger.Verbose($"Writing to {targetName}...");
if (context.WriteProgressInterval != 0)
{
var i = 0L;
rows = rows.Select(
a =>
{
i++;
if (i % context.WriteProgressInterval == 0)
{
context.Logger.Information($"Wrote {i} items.");
}
return(a);
});
}
try
{
var result = await this.target.WriteRowsAsync(context, rows, upsert);
context.Logger.Verbose($"Wrote {result} rows to {context.GetDisplayName(this.target)}");
return(result);
}
catch (Exception e)
{
context.Logger.Error($"An error occurred while writing to {targetName}: {e.Message}.");
return(0);
}
}
/// <summary>
/// The get data async.
/// </summary>
/// <param name="context">
/// The context.
/// </param>
/// <param name="multiPartQuery">
/// The multi part query.
/// </param>
/// <returns>
/// The <see cref="Task"/>.
/// </returns>
internal override IAsyncEnumerable <Row> GetRows(IInternalExecutionContext context, [NotNull] IMultiPartQuery multiPartQuery)
{
var fieldReplacer = GenericVisitor.Create((SourceFieldExpression e) => CustomExpression.MakeField(e.SourceName, e.FieldName, e.Type));
var query =
multiPartQuery.WildcardAliases.Contains(this.alias) ?
new Query(
fieldReplacer.Visit(multiPartQuery.FilterExpression),
multiPartQuery.OrderByExpressions.Select(o => new OrderByExpression(fieldReplacer.Visit(o.Expression), o.Ascending)),
multiPartQuery.Count)
: new Query(
multiPartQuery.Fields.Where(f => f.SourceAlias == this.alias).Select(f => f.FieldName),
fieldReplacer.Visit(multiPartQuery.FilterExpression),
multiPartQuery.OrderByExpressions.Select(o => new OrderByExpression(fieldReplacer.Visit(o.Expression), o.Ascending)),
multiPartQuery.Count);
var rowBuilder = new RowBuilder(this.alias);
return(context
.CreateAsyncEnumerable(async() => (await this.selectPlan.ExecuteAsync(context)).QueryResults.First().Rows)
.Select(rowBuilder.Attach)
.Where(query.GetFilter(context)?.GetRowFilter())
.OrderBy(query.GetSortOrders(context)));
}
/// <summary>
/// Gets the joined data.
/// </summary>
/// <param name="context">
/// The execution context.
/// </param>
/// <param name="query">
/// The query.
/// </param>
/// <returns>
/// A data set containing the rows retrieved from the source.
/// </returns>
protected override IAsyncEnumerable <Row> GetRows(IInternalExecutionContext context, JoinQuery query)
{
throw new NotImplementedException();
}
/// <summary>
/// Executes the plan.
/// </summary>
/// <param name="context">
/// The context.
/// </param>
/// <returns>
/// The <see cref="ExecuteResult"/>.
/// </returns>
public Task <ExecuteResult> ExecuteAsync([NotNull] IInternalExecutionContext context)
{
context.RegisterDefault(this.setting, this.functionName, this.valueFactory(context));
return(Task.FromResult(new ExecuteResult()));
}
/// <summary>
/// Retrieves the data from the source as an <see cref="IAsyncEnumerable{T}"/>.
/// </summary>
/// <param name="context">
/// The context.
/// </param>
/// <param name="multiPartQuery">
/// The query expression. Can be <c>null</c>.
/// </param>
/// <returns>
/// A task returning the data set.
/// </returns>
public IAsyncEnumerable <Row> GetRows(IInternalExecutionContext context, IMultiPartQuery multiPartQuery)
{
return(this.GetRows(context, this.CreateJoinQuery(context, multiPartQuery)));
}
/// <summary>
/// Retrieves the data from the source as an <see cref="IAsyncEnumerable{T}"/>.
/// </summary>
/// <param name="context">
/// The context.
/// </param>
/// <param name="query">
/// The query expression.
/// </param>
/// <returns>
/// A task returning the rows.
/// </returns>
protected abstract IAsyncEnumerable <Row> GetRows(IInternalExecutionContext context, JoinQuery query);
public async Task <ExecuteResult> ExecuteAsync(IInternalExecutionContext context)
{
await this.evaluateVariable(context);
return(new ExecuteResult());
}
/// <summary>
/// The get rows.
/// </summary>
/// <param name="context">
/// The context.
/// </param>
/// <param name="multiPartQuery">
/// The multi-part query.
/// </param>
/// <returns>
/// The <see cref="IAsyncEnumerable{Row}"/>.
/// </returns>
internal override IAsyncEnumerable <Row> GetRows([NotNull] IInternalExecutionContext context, [NotNull] IMultiPartQuery multiPartQuery)
{
var functionName = context.GetDisplayName(this.dataSource);
var fieldReplacer = GenericVisitor.Create((SourceFieldExpression e) => CustomExpression.MakeField(e.SourceName, e.FieldName, e.Type));
var query =
multiPartQuery.WildcardAliases.Contains(this.alias) ?
new Query(
fieldReplacer.Visit(multiPartQuery.FilterExpression),
multiPartQuery.OrderByExpressions.Select(o => new OrderByExpression(fieldReplacer.Visit(o.Expression), o.Ascending)),
multiPartQuery.Count)
: new Query(
multiPartQuery.Fields.Where(f => f.SourceAlias == this.alias).Select(f => f.FieldName),
fieldReplacer.Visit(multiPartQuery.FilterExpression),
multiPartQuery.OrderByExpressions.Select(o => new OrderByExpression(fieldReplacer.Visit(o.Expression), o.Ascending)),
multiPartQuery.Count);
Expression unsupportedFilters = null;
IOrderByExpression[] unsupportedOrderByExpressions = null;
// ReSharper disable once SuspiciousTypeConversion.Global, implemented in other assemblies.
var expressionSupport = this.dataSource as IDataSourceFilterSupport;
if (expressionSupport != null)
{
var parts = query.FilterExpression.SplitByAndExpressions().Cast <CompareExpression>().ToArray();
var filters = parts.ToLookup(p => expressionSupport.SupportsExpression(p), e => (Expression)e);
var supportedFilter = filters[true].DefaultIfEmpty().Aggregate(Expression.AndAlso);
unsupportedFilters = filters[false].DefaultIfEmpty().Aggregate(Expression.AndAlso);
if (!query.RetrieveAllFields)
{
query = new Query(query.Fields.Concat(unsupportedFilters.GetFields().Select(u => u.FieldName)).Distinct(), supportedFilter, query.OrderByExpressions, query.Count);
}
if (unsupportedFilters != null)
{
context.Logger.Warning($"Data source {functionName} {this.alias} has unsupported filter {unsupportedFilters}. This could impact performance.");
}
}
// ReSharper disable once SuspiciousTypeConversion.Global, implemented in other assemblies.
var orderBySupport = this.dataSource as IDataSourceOrderBySupport;
if (orderBySupport != null && !orderBySupport.SupportsOrderBy(query.OrderByExpressions))
{
unsupportedOrderByExpressions = query.OrderByExpressions.ToArray();
if (!query.RetrieveAllFields)
{
query = new Query(query.Fields.Concat(unsupportedOrderByExpressions.SelectMany(e => e.Expression.GetFields().Select(f => f.FieldName))).Distinct(), query.FilterExpression, null, query.Count);
}
context.Logger.Warning($"Data source {functionName} {this.alias} has unsupported ORDER BY {string.Join(", ", unsupportedOrderByExpressions.Select(u => u.Expression + " " + (u.Ascending ? "ASC" : "DESC")))}. This could impact performance.");
}
var sourceName = functionName + (query.FilterExpression == null ? string.Empty : $" with query '{query.FilterExpression}'");
context.Logger.Verbose($"Retrieving data from {sourceName}.");
try
{
var result = this.dataSource.GetRows(context, new RowBuilder(this.alias), query).AfterLastElement(count => context.Logger.Verbose($"Retrieved {count} items from {sourceName}."));
if (unsupportedFilters != null)
{
result = result.Where(unsupportedFilters.GetRowFilter());
}
if (unsupportedOrderByExpressions != null)
{
result.OrderBy(unsupportedOrderByExpressions);
}
return(result);
}
catch (Exception e)
{
context.Logger.Error($"An error occurred while querying {sourceName}: {e.Message}.");
return(context.CreateEmptyAsyncEnumerable <Row>());
}
}
/// <summary>
/// Combines the left and right result sets.
/// </summary>
/// <param name="context">
/// The context.
/// </param>
/// <param name="leftData">
/// The left result set.
/// </param>
/// <param name="rightData">
/// The right result set.
/// </param>
/// <param name="rightQuery">
/// The query for the right side.
/// </param>
/// <param name="rowBuilder">
/// The row builder.
/// </param>
/// <returns>
/// The <see cref="IAsyncEnumerable{Row}"/>.
/// </returns>
protected abstract IAsyncEnumerable <Row> CombineResults(IInternalExecutionContext context, IAsyncReadOnlyCollection <Row> leftData, IAsyncReadOnlyCollection <Row> rightData, MultiPartQuery rightQuery, RowBuilder rowBuilder);
/// <summary>
/// Combines the left and right result sets.
/// </summary>
/// <param name="context">
/// The context.
/// </param>
/// <param name="leftData">
/// The left result set.
/// </param>
/// <param name="rightData">
/// The right result set.
/// </param>
/// <param name="rightQuery">
/// The query for the right side.
/// </param>
/// <param name="rowBuilder">
/// The row builder.
/// </param>
/// <returns>
/// The <see cref="IAsyncEnumerable{Row}"/>.
/// </returns>
protected override IAsyncEnumerable <Row> CombineResults(IInternalExecutionContext context, IAsyncReadOnlyCollection <Row> leftData, IAsyncReadOnlyCollection <Row> rightData, MultiPartQuery rightQuery, [NotNull] RowBuilder rowBuilder)
{
return(this.RightFactory != null
? leftData.CrossApply(row => this.RightFactory(context, row).GetRows(context, rightQuery), rowBuilder.CombineRows)
: leftData.CrossApply(row => rightData, rowBuilder.CombineRows));
}
public async Task <ExecuteResult> ExecuteAsync(IInternalExecutionContext context)
{
return(new ExecuteResult(new Job(context, this.name, this.plan, await this.triggersFactory(context))));
}
/// <summary> /// The get data async. /// </summary> /// <param name="context"> /// The context. /// </param> /// <param name="query"> /// The query. /// </param> /// <returns> /// The <see cref="Task"/>. /// </returns> internal abstract IAsyncEnumerable <Row> GetRows(IInternalExecutionContext context, IMultiPartQuery query);
