/// <summary> /// Writes the rowsToWrite to the specified target. /// </summary> /// <param name="context"> /// The context. /// </param> /// <param name="rowsToWrite"> /// The rowsToWrite. /// </param> /// <param name="upsert"> /// True to also update records, false to insert. /// </param> /// <returns> /// The <see cref="Task"/>. /// </returns> public async Task <long> WriteRowsAsync([NotNull] IExecutionContext context, IAsyncEnumerable <Row> rowsToWrite, bool upsert) { var builder = context.CreateBuilder <Row>(); if (this.rows == null) { this.rows = await rowsToWrite.MaterializeAsync(); return(this.rows.Count); } var originalCount = this.rows.Count; if (upsert) { await builder.AddAsync(this.rows.Union(rowsToWrite, new RowUniqueIdComparer())); } else { await builder.AddAsync(this.rows).ConfigureAwait(false); await builder.AddAsync(rowsToWrite).ConfigureAwait(false); } this.rows = await builder.BuildAsync().ConfigureAwait(false); return(this.rows.Count - originalCount); }
/// <summary> /// Moves to the next batch. Implemented as a state machine. /// </summary> /// <returns> /// <c>true</c> if another batch is available, <c>false</c> otherwise. /// </returns> protected override async Task <IEnumerator <IAsyncGrouping <TSource, TKey> > > OnNextBatchAsync() { switch (this.state) { case 0: // Initial, sort the source, and return the first groupings. Comparison <SourceKey <TSource, TKey> > comparison = (first, second) => this.comparer.Compare(first.Key, second.Key); this.sorted = await this.source.Policy.SortAsync(this.source, comparison).ConfigureAwait(false); this.enumerator = this.sorted.GetAsyncEnumerator(); this.state = 1; return(this.EnumerateGroupings()); case 1: // Continue, return the next groupings. if (!await this.enumerator.NextBatchAsync().ConfigureAwait(false)) { this.state = 2; return(this.offset == this.lastOffset ? null : GroupByEnumerator <TSource, TKey> .EnumerateItem(new AsyncGrouping <TSource, TKey>(this.lastKey, this.sorted, this.lastOffset, this.offset - this.lastOffset))); } return(this.EnumerateGroupings()); case 2: // Done, no more groupings available. return(null); case 3: // Disposed, throw error. throw new ObjectDisposedException(this.GetType().ToString()); default: // Shouldn't happen. throw new InvalidOperationException($"Invalid state: {this.state}."); } }
/// <summary> /// Initializes a new instance of the <see cref="JoinEnumerator{TLeft,TRight,TKey,TResult}"/> class. /// </summary> /// <param name="isLeftJoin"> /// The is left join. /// </param> /// <param name="isPreSorted"> /// True if both <see cref="IAsyncEnumerable{T}"/>s are already sorted. /// </param> /// <param name="left"> /// The left. /// </param> /// <param name="right"> /// The right. /// </param> /// <param name="leftKey"> /// The left key. /// </param> /// <param name="joinOperator"> /// The operator to use when joining. /// </param> /// <param name="rightKey"> /// The right key. /// </param> /// <param name="resultFilter"> /// The result filter. /// </param> /// <param name="resultSelector"> /// The result selector. /// </param> /// <param name="keyComparer"> /// The key comparer. /// </param> public JoinEnumerator(bool isLeftJoin, bool isPreSorted, IAsyncEnumerable <TLeft> left, IAsyncEnumerable <TRight> right, Func <TLeft, TKey> leftKey, ExpressionType joinOperator, Func <TRight, TKey> rightKey, Func <TLeft, TRight, bool> resultFilter, Func <TLeft, TRight, TResult> resultSelector, [NotNull] IComparer <TKey> keyComparer) { this.left = left; this.right = right; this.leftKey = leftKey; this.joinOperator = joinOperator; this.rightKey = rightKey; this.resultSelector = resultSelector; this.keyComparison = keyComparer.Compare; this.isLeftJoin = isLeftJoin; this.resultFilter = resultFilter; if (this.joinOperator == ExpressionType.GreaterThan) { // Invert the comparison order and do a less than or equal join instead. this.joinOperator = ExpressionType.LessThanOrEqual; this.keyComparison = (x, y) => - keyComparer.Compare(x, y); } if (this.joinOperator == ExpressionType.GreaterThanOrEqual) { // Invert the comparison order and do a less than join instead. this.joinOperator = ExpressionType.LessThan; this.keyComparison = (x, y) => - keyComparer.Compare(x, y); } this.materializedRight = this.right as IAsyncReadOnlyCollection <TRight>; if (this.materializedRight != null && isPreSorted) { this.leftEnumerator = this.left.GetAsyncEnumerator(); this.rightEnumerator = this.materializedRight.GetAsyncEnumerator(); this.IsSynchronous = this.leftEnumerator.IsSynchronous && this.rightEnumerator.IsSynchronous; } }
/* * /// <summary> * /// Retrieves the data from the source. * /// </summary> * /// <param name="context"> * /// The context. * /// </param> * /// <param name="query"> * /// The query expression. Can be <c>null</c>. * /// </param> * /// <returns> * /// A task returning the data set. * /// </returns> * public async Task<DataSet> GetDataAsync(IExecutionContext context, IQuery query) * { * * var leftRows = await context.MaterializeAsync(await this.GetDataAsync(context, this.left, query, query.GetFilter(context), Enumerable.Empty<OrderByExpression>()).ConfigureAwait(false)).ConfigureAwait(false); * * if (leftRows.Count == 0) * { * return DataSet.Empty(); * } * * var sortOrders = new List<OrderByExpression>(); * var filter = await this.CreateJoinFilterAsync(context, leftRows, sortOrders); * var rightRows = await context.MaterializeAsync(await this.GetDataAsync(context, this.right, query, filter, sortOrders).ConfigureAwait(false)); * * if (rightRows.Count == 0) * { * return this.isInnerJoin ? DataSet.Empty() : DataSet.FromEnumerable(rightRows); * } * * var joinFilter = this.GetFilter(context); * * return null; * } */ /// <summary> /// Creates the join filter. /// </summary> /// <param name="context"> /// The execution context. /// </param> /// <param name="leftRows"> /// The left rows. /// </param> /// <param name="sortOrders"> /// Will be filled with the sort orders for this join. /// </param> /// <returns> /// A task containing the the filter expression. /// </returns> private Task <Expression> CreateJoinFilterAsync(IExecutionContext context, IAsyncReadOnlyCollection <Row> leftRows, ICollection <OrderByExpression> sortOrders) { var filter = (Expression)null; // await this.GetFilter(context, null).ToRangedExpressionAsync(leftRows, this.right.Aliases); return(Task.FromResult(new GenericVisitor { (GenericVisitor visitor, BinaryExpression node) => { if (node.NodeType != ExpressionType.And && node.NodeType != ExpressionType.AndAlso) { return null; } Expression leftSide, rightSide; if (((leftSide = visitor.Visit(node.Left)) as ConstantExpression)?.Value?.Equals(true) ?? false) { return visitor.Visit(node.Right); } if (((rightSide = visitor.Visit(node.Right)) as ConstantExpression)?.Value?.Equals(true) ?? false) { return visitor.Visit(node.Left); } return Expression.MakeBinary(ExpressionType.AndAlso, leftSide, rightSide); }, (RangeExpression node) => node.Type == typeof(bool) && object.Equals(node.Min, false) && object.Equals(node.Max, true) ? (Expression)Expression.Constant(true) : node, (CompareExpression node) => { var field = node.Left as SourceFieldExpression; var range = node.Right as RangeExpression; if (field == null || range == null) { return null; } switch (node.CompareType) { case ExpressionType.GreaterThan: case ExpressionType.GreaterThanOrEqual: case ExpressionType.LessThan: case ExpressionType.LessThanOrEqual: sortOrders.Add( new OrderByExpression( CustomExpression.MakeSourceField(field.SourceName, field.FieldName), node.CompareType == ExpressionType.GreaterThan || node.CompareType == ExpressionType.GreaterThanOrEqual)); break; } return Expression.AndAlso( CustomExpression.MakeCompare(ExpressionType.GreaterThanOrEqual, field, Expression.Constant(range.Min, range.Type)), CustomExpression.MakeCompare(ExpressionType.LessThanOrEqual, field, Expression.Constant(range.Max, range.Type))); }, }.Visit(filter))); }
/// <summary> /// Moves to the next batch. Implemented as a state machine. /// </summary> /// <returns> /// <c>true</c> if another batch is available, <c>false</c> otherwise. /// </returns> protected override async Task <IEnumerator <IAsyncEnumerable <TSource> > > OnNextBatchAsync() { switch (this.state) { case 0: // Initial, materialize the source, and return the first groupings. this.materialized = await this.source.Policy.MaterializeAsync(this.source).ConfigureAwait(false); this.enumerator = this.materialized.GetAsyncEnumerator(); return(this.EnumerateBatches()); case 1: // Continue, return the next groupings. if (!await this.enumerator.NextBatchAsync().ConfigureAwait(false)) { this.state = 2; return(this.startOffset == this.currentOffset ? null : BatchesEnumerator <TSource> .EnumerateItem(new Batch <TSource>(this.materialized, this.startOffset, this.currentOffset - this.startOffset))); } return(this.EnumerateBatches()); case 2: // Done, no more batches available. return(null); case 3: // Disposed, throw error. throw new ObjectDisposedException(this.GetType().ToString()); default: // Shouldn't happen. throw new InvalidOperationException($"Invalid state: {this.state}."); } }
/// <summary> /// Resets all fields to null, and sets the state to 'disposed'. /// </summary> /// <param name="disposing"> /// The disposing. /// </param> protected override void Dispose(bool disposing) { base.Dispose(disposing); this.state = 3; this.source = null; this.policy = null; this.materialized = null; this.enumerator = null; }
/// <summary> /// Initializes a new instance of the <see cref="BatchesEnumerator{TSource}"/> class. /// </summary> /// <param name="source"> /// The source. /// </param> /// <param name="batchSize"> /// The batch size. /// </param> public BatchesEnumerator(IAsyncEnumerable <TSource> source, long batchSize) { this.source = source; this.batchSize = batchSize; this.materialized = source as IAsyncReadOnlyCollection <TSource>; if (this.materialized != null) { this.enumerator = this.materialized.GetAsyncEnumerator(); this.IsSynchronous = this.enumerator.IsSynchronous; } }
/// <summary> /// Initializes a new instance of the <see cref="CrossJoinEnumerator{TLeft,TRight,TResult}"/> class. /// </summary> /// <param name="left"> /// The left part of the union. /// </param> /// <param name="right"> /// The right part of the union. /// </param> /// <param name="resultSelector"> /// The result Selector. /// </param> public CrossJoinEnumerator([NotNull] IAsyncEnumerable <TLeft> left, IAsyncEnumerable <TRight> right, Func <TLeft, TRight, TResult> resultSelector) { this.resultSelector = resultSelector; this.leftEnumerator = left.GetAsyncEnumerator(); this.right = right; this.materializedRight = this.right as IAsyncReadOnlyCollection <TRight>; if (this.materializedRight != null) { this.rightEnumerator = this.materializedRight.GetAsyncEnumerator(); this.IsSynchronous = this.leftEnumerator.IsSynchronous && this.rightEnumerator.IsSynchronous; } }
/// <summary> /// Resets all fields to null, and sets the state to 'disposed'. /// </summary> /// <param name="disposing"> /// The disposing. /// </param> protected override void Dispose(bool disposing) { base.Dispose(disposing); this.state = 3; this.enumerator?.Dispose(); this.source = null; this.keySelector = null; this.comparer = null; this.sorted = null; this.enumerator = null; this.lastKey = default(TKey); }
/// <summary> /// Disposes the <see cref="IAsyncEnumerator{T}"/>. /// </summary> /// <param name="disposing"> /// The disposing. /// </param> protected override void Dispose(bool disposing) { base.Dispose(disposing); this.state = 3; this.leftEnumerator?.Dispose(); this.rightEnumerator?.Dispose(); this.resultSelector = null; this.leftEnumerator = null; this.rightEnumerator = null; this.materializedRight = null; this.right = null; }
/// <summary> /// Gets called when the next batch is needed. /// </summary> /// <returns> /// A task returning an <see cref="IEnumerator{T}"/> containing the next batch, of <c>null</c> when all data is /// enumerated. /// </returns> protected override async Task <IEnumerator <TResult> > OnNextBatchAsync() { switch (this.state) { case 0: Func <Task> setLeft = async() => this.leftEnumerator = (await this.left.SortAsync((item1, item2) => this.keyComparison(this.leftKey(item1), this.leftKey(item2))).ConfigureAwait(false)).GetAsyncEnumerator(); Func <Task> setRight = async() => this.rightEnumerator = (this.materializedRight = await this.right.SortAsync((item1, item2) => this.keyComparison(this.rightKey(item1), this.rightKey(item2))).ConfigureAwait(false)).GetAsyncEnumerator(); await Task.WhenAll(setLeft(), setRight()).ConfigureAwait(false); return(this.EnumerateItems()); case 1: // Next batch for the left side. if (!await this.leftEnumerator.NextBatchAsync().ConfigureAwait(false)) { this.state = 3; return(null); } return(this.EnumerateItems()); case 2: // Next batch for the right side. if (!await this.rightEnumerator.NextBatchAsync().ConfigureAwait(false)) { if (this.isLeftJoin && this.itemsReturned == 0) { return(JoinEnumerator <TLeft, TRight, TKey, TResult> .EnumerateItem(this.resultSelector(this.leftEnumerator.Current, default(TRight)))); } this.ResetRight(); } return(this.EnumerateItems()); case 3: // Done. return(null); case 4: // Disposed. throw new ObjectDisposedException(this.GetType().ToString()); default: throw new InvalidOperationException($"Invalid state: {this.state}."); } }
/// <summary> /// Gets the rows for the join. /// </summary> /// <param name="context"> /// The context. /// </param> /// <param name="multiPartQuery"> /// The multi part query. /// </param> /// <returns> /// The <see cref="IAsyncEnumerable{Row}"/>. /// </returns> internal override IAsyncEnumerable <Row> GetRows(IInternalExecutionContext context, [NotNull] IMultiPartQuery multiPartQuery) { var rowBuilder = new RowBuilder(); //// Build the left part by filtering by parts that contain the fields of the left side. var leftQuery = new MultiPartQuery { Fields = multiPartQuery.Fields.Where(f => this.left.Aliases.Contains(f.SourceAlias)).Union(this.extraFields), FilterExpression = multiPartQuery.FilterExpression.FilterByAliases(this.left.Aliases), WildcardAliases = multiPartQuery.WildcardAliases.Intersect(this.left.Aliases), }; //// Create the enumerable. return(context.CreateAsyncEnumerable( async() => { //// Retrieve the records from the left side. var leftData = await this.left.GetRows(context, leftQuery).MaterializeAsync().ConfigureAwait(false); var rightQuery = new MultiPartQuery { Fields = multiPartQuery.Fields.Where(f => this.right.Aliases.Contains(f.SourceAlias)), FilterExpression = ApplyBase.RangesToJoinFilter(await this.FindRangesAsync(context, multiPartQuery.FilterExpression, leftData)), WildcardAliases = multiPartQuery.WildcardAliases.Intersect(this.right.Aliases), }; IAsyncReadOnlyCollection <Row> rightData = null; if (this.RightFactory == null) { rightData = await this.right.GetRows(context, rightQuery).MaterializeAsync().ConfigureAwait(false); } var collection = this.CombineResults(context, leftData, rightData, rightQuery, rowBuilder); collection = await collection.MaterializeAsync().ConfigureAwait(false); return collection; }) .Where(multiPartQuery.FilterExpression.GetRowFilter()) .OrderBy(multiPartQuery.OrderByExpressions) .AfterLastElement(count => context.Logger.Verbose($"{this.GetType().Name} returned {count} records."))); }
/// <summary> /// Disposes the <see cref="IAsyncEnumerator{T}"/>. /// </summary> /// <param name="disposing"> /// The disposing. /// </param> protected override void Dispose(bool disposing) { base.Dispose(disposing); this.state = 4; this.leftEnumerator?.Dispose(); this.rightEnumerator?.Dispose(); this.left = null; this.right = null; this.leftKey = null; this.rightKey = null; this.resultSelector = null; this.keyComparison = null; this.leftEnumerator = null; this.rightEnumerator = null; this.materializedRight = null; this.resultFilter = null; }
/// <summary> /// Gets called when the next batch is needed. /// </summary> /// <returns> /// A task returning an <see cref="IEnumerator{T}"/> containing the next batch, of <c>null</c> when all data is /// enumerated. /// </returns> protected override async Task <IEnumerator <TResult> > OnNextBatchAsync() { switch (this.state) { case 0: // Right side was not materialized, materialize it and start enumeration. this.materializedRight = await this.right.MaterializeAsync().ConfigureAwait(false); this.rightEnumerator = this.materializedRight.GetAsyncEnumerator(); return(this.EnumerateItems()); case 1: // Next batch for the left side. if (await this.leftEnumerator.NextBatchAsync().ConfigureAwait(false)) { return(this.EnumerateItems()); } this.state = 3; return(null); case 2: // Next batch for the right side. if (!await this.rightEnumerator.NextBatchAsync().ConfigureAwait(false)) { this.rightEnumerator.Dispose(); this.rightEnumerator = this.materializedRight.GetAsyncEnumerator(); this.stillEnumerating = false; } return(this.EnumerateItems()); case 3: // Done. return(null); case 4: // Disposed. throw new ObjectDisposedException(this.GetType().ToString()); default: throw new InvalidOperationException($"Invalid state: {this.state}."); } }
/// <summary> /// Moves to the next batch. Implemented as a state machine. /// </summary> /// <returns> /// <c>true</c> if another batch is available, <c>false</c> otherwise. /// </returns> protected override async Task <IEnumerator <IAsyncEnumerable <TSource> > > OnNextBatchAsync() { switch (this.state) { case 0: // Initial, sort the source, and return the first batches. Comparison <TSource> comparison = (x, y) => this.comparer.Compare(this.valueSelector(x), this.valueSelector(y)); this.materialized = await this.policy.SortAsync(this.source, comparison).ConfigureAwait(false); this.enumerator = this.materialized.GetAsyncEnumerator(); // Enumerate batches, and return to state 1. this.state = 1; return(this.EnumerateBatches()); case 1: // Continue, return the next batches. if (!await this.enumerator.NextBatchAsync().ConfigureAwait(false)) { // We're done, check if there still is a batch left. this.state = 2; var count = this.offset % this.batchSize; return(count == 0 ? null : ValueBatchesEnumerator <TSource, TValue> .EnumerateItem(new Batch <TSource>(this.materialized, this.offset - count, count))); } return(this.EnumerateBatches()); case 2: // Done, no more groupings available. return(null); case 3: // Disposed, throw error. throw new ObjectDisposedException(this.GetType().ToString()); default: // Shouldn't happen. throw new InvalidOperationException($"Invalid state: {this.state}."); } }
/// <summary> /// Combines the results of the left and right parts into the query result. /// </summary> /// <param name="joinsParts"> /// An array of array of compare expressions. /// </param> /// <param name="rowBuilder"> /// The row builder. /// </param> /// <param name="leftData"> /// The left data. /// </param> /// <param name="rightData"> /// The right data. /// </param> /// <returns> /// The <see cref="IAsyncEnumerable{Row}"/>. /// </returns> protected override IAsyncEnumerable <Row> CombineResults([NotNull] CompareExpression[][] joinsParts, RowBuilder rowBuilder, IAsyncReadOnlyCollection <Row> leftData, IAsyncReadOnlyCollection <Row> rightData) { return(joinsParts .Select(joinPart => leftData.Join( rightData, joinPart[0].Left.GetRowExpression <Row>(), joinPart[0].CompareType, joinPart[0].Right.GetRowExpression <Row>(), joinPart.Skip(1).DefaultIfEmpty().Aggregate <Expression>(Expression.AndAlso).GetJoinFunction(this.Left), rowBuilder.CombineRows, joinPart[0].Comparer)) .Aggregate((result, joinResult) => result.Union(joinResult, RowIdComparer.Default))); }
/// <summary> /// Finds the ranges in an expression using the already retrieved rows. /// </summary> /// <param name="context"> /// The context. /// </param> /// <param name="expression"> /// The expression. /// </param> /// <param name="rows"> /// The rows. /// </param> /// <returns> /// A <see cref="Task"/> returning the <see cref="Expression"/> containing ranges. /// </returns> private async Task <Expression> FindRangesAsync(IExecutionContext context, Expression expression, IAsyncReadOnlyCollection <Row> rows) { var ranges = await expression.SplitByOrExpressions().ToRangedExpressionAsync(rows, this.Right.Aliases).ConfigureAwait(false); return(ranges .Select(r => r.SimplifyExpression(context)) .Where(r => !object.Equals((r as ConstantExpression)?.Value, false)) .DefaultIfEmpty().Aggregate(Expression.OrElse).SimplifyRanges()); }
/// <summary> /// Combines the results of the left and right parts into the query result. /// </summary> /// <param name="joinsParts"> /// An array of array of compare expressions. /// </param> /// <param name="rowBuilder"> /// The row builder. /// </param> /// <param name="leftData"> /// The left data. /// </param> /// <param name="rightData"> /// The right data. /// </param> /// <returns> /// The <see cref="IAsyncEnumerable{Row}"/>. /// </returns> protected abstract IAsyncEnumerable <Row> CombineResults(CompareExpression[][] joinsParts, RowBuilder rowBuilder, IAsyncReadOnlyCollection <Row> leftData, IAsyncReadOnlyCollection <Row> rightData);
/// <summary> /// Initializes a new instance of the <see cref="Batch{T}"/> class. /// </summary> /// <param name="materialized"> /// The materialized. /// </param> /// <param name="start"> /// The start. /// </param> /// <param name="count"> /// The count. /// </param> public Batch(IAsyncReadOnlyCollection <T> materialized, long start, long count) { this.materialized = materialized; this.start = start; this.Count = count; }
/// <summary> /// Combines the left and right result sets. /// </summary> /// <param name="context"> /// The context. /// </param> /// <param name="leftData"> /// The left result set. /// </param> /// <param name="rightData"> /// The right result set. /// </param> /// <param name="rightQuery"> /// The query for the right side. /// </param> /// <param name="rowBuilder"> /// The row builder. /// </param> /// <returns> /// The <see cref="IAsyncEnumerable{Row}"/>. /// </returns> protected override IAsyncEnumerable <Row> CombineResults(IInternalExecutionContext context, IAsyncReadOnlyCollection <Row> leftData, IAsyncReadOnlyCollection <Row> rightData, MultiPartQuery rightQuery, [NotNull] RowBuilder rowBuilder) { return(this.RightFactory != null ? leftData.CrossApply(row => this.RightFactory(context, row).GetRows(context, rightQuery), rowBuilder.CombineRows) : leftData.CrossApply(row => rightData, rowBuilder.CombineRows)); }
/// <summary> /// Combines the left and right result sets. /// </summary> /// <param name="context"> /// The context. /// </param> /// <param name="leftData"> /// The left result set. /// </param> /// <param name="rightData"> /// The right result set. /// </param> /// <param name="rightQuery"> /// The query for the right side. /// </param> /// <param name="rowBuilder"> /// The row builder. /// </param> /// <returns> /// The <see cref="IAsyncEnumerable{Row}"/>. /// </returns> protected abstract IAsyncEnumerable <Row> CombineResults(IInternalExecutionContext context, IAsyncReadOnlyCollection <Row> leftData, IAsyncReadOnlyCollection <Row> rightData, MultiPartQuery rightQuery, RowBuilder rowBuilder);
/// <summary> /// Initializes a new instance of the <see cref="MaterializedAsyncEnumerableVisualizer{T}"/> class. /// </summary> /// <param name="readOnlyCollection"> /// The enumerable. /// </param> public MaterializedAsyncEnumerableVisualizer([NotNull] IAsyncReadOnlyCollection <T> readOnlyCollection) : base(readOnlyCollection) { this.Count = readOnlyCollection.Count; this.Policy = readOnlyCollection.Policy; }
/// <summary> /// Initializes a new instance of the <see cref="SourceKeyAdapter"/> class. /// </summary> /// <param name="inner"> /// The inner collection. /// </param> public SourceKeyAdapter(IAsyncReadOnlyCollection <SourceKey <TSource, TKey> > inner) { this.inner = inner; }
/// <summary> /// Initializes a new instance of the <see cref="AsyncGrouping{TSource,TKey}"/> class. /// </summary> /// <param name="key"> /// The key. /// </param> /// <param name="items"> /// The inner. /// </param> /// <param name="start"> /// The offset. /// </param> /// <param name="count"> /// The number of inner. /// </param> public AsyncGrouping(TKey key, IAsyncReadOnlyCollection <SourceKey <TSource, TKey> > items, long start, long count) : base(new SourceKeyAdapter(items), start, count) { this.Key = key; }