private void ExtractRelationsAndPredicates(BoundRelation node, out List <BoundRelation> relations, out List <BoundExpression> predicates)
{
relations = new List <BoundRelation>();
predicates = new List <BoundExpression>();
var stack = new Stack <BoundRelation>();
stack.Push(node);
while (stack.Count > 0)
{
var current = stack.Pop();
var join = current as BoundJoinRelation;
if (join == null || join.JoinType != BoundJoinType.Inner)
{
// NOTE: We generally want to rewrite joins that we can't extract
// ourselves. However, we've to be careful not rewrite the
// node that we started from -- otherwise we stack overflow.
var rewrittenCurrent = current == node ? current : RewriteRelation(current);
relations.Add(rewrittenCurrent);
}
else
{
if (join.Condition != null)
{
var conjunctions = Expression.SplitConjunctions(join.Condition);
predicates.AddRange(conjunctions);
}
stack.Push(join.Right);
stack.Push(join.Left);
}
}
}
public Subquery(SubqueryKind kind, ValueSlot valueSlot, BoundRelation relation, BoundExpression passthru)
{
Kind = kind;
ValueSlot = valueSlot;
Relation = relation;
Passthru = passthru;
}
public static bool ContainsSubquery(BoundRelation node)
{
var checker = new SubqueryChecker();
checker.VisitRelation(node);
return(checker.SubqueryFound);
}
private static BoundRelation Optimize(BoundRelation relation)
{
foreach (var step in GetOptimizationSteps())
{
relation = step.RewriteRelation(relation);
}
return(relation);
}
public override void VisitRelation(BoundRelation node)
{
if (SubqueryFound)
{
return;
}
base.VisitRelation(node);
}
private static BoundRelation RewriteConjunctions(BoundRelation input, BoundExpression condition)
{
var current = input;
var scalarPredicates = new List <BoundExpression>();
var conjunctions = Expression.SplitConjunctions(condition);
foreach (var conjunction in conjunctions)
{
BoundExistsSubselect exists;
bool isNegated;
if (TryGetExistsSubselect(conjunction, out exists, out isNegated))
{
var joinType = isNegated
? BoundJoinType.LeftAntiSemi
: BoundJoinType.LeftSemi;
current = new BoundJoinRelation(joinType, current, exists.Relation, null, null, null);
}
else if (Expression.IsDisjunction(conjunction))
{
var relation = RewriteDisjunctions(conjunction);
if (relation != null)
{
current = new BoundJoinRelation(BoundJoinType.LeftSemi, current, relation, null, null, null);
}
else
{
scalarPredicates.Add(conjunction);
}
}
else
{
scalarPredicates.Add(conjunction);
}
}
// If we haven't done anything, simply return null to indicate to our
// caller that the condition only contained scalars.
if (current == input)
{
return(null);
}
// If we have no scalar predicates left, it means the condition only
// contained EXISTs queries, so we can return the current node.
if (scalarPredicates.Count == 0)
{
return(current);
}
// Othwerwise We add a filter for the scalars.
var predicate = Expression.And(scalarPredicates);
return(new BoundFilterRelation(current, predicate));
}
public override BoundRelation RewriteRelation(BoundRelation node)
{
if (_recorder == null)
{
_recorder = new ValueSlotRecorder();
_recorder.Record(node.GetOutputValues());
}
return(base.RewriteRelation(node));
}
private Iterator BuildRelation(BoundRelation relation)
{
switch (relation.Kind)
{
case BoundNodeKind.ConstantRelation:
return(BuildConstant());
case BoundNodeKind.TableRelation:
return(BuildTable((BoundTableRelation)relation));
case BoundNodeKind.JoinRelation:
return(BuildJoin((BoundJoinRelation)relation));
case BoundNodeKind.HashMatchRelation:
return(BuildHashMatch((BoundHashMatchRelation)relation));
case BoundNodeKind.FilterRelation:
return(BuildFilter((BoundFilterRelation)relation));
case BoundNodeKind.ComputeRelation:
return(BuildCompute((BoundComputeRelation)relation));
case BoundNodeKind.TopRelation:
return(BuildTop((BoundTopRelation)relation));
case BoundNodeKind.SortRelation:
return(BuildSort((BoundSortRelation)relation));
case BoundNodeKind.ConcatenationRelation:
return(BuildConcatenationRelation((BoundConcatenationRelation)relation));
case BoundNodeKind.StreamAggregatesRelation:
return(BuildStreamAggregatesRelation((BoundStreamAggregatesRelation)relation));
case BoundNodeKind.ProjectRelation:
return(BuildProject((BoundProjectRelation)relation));
case BoundNodeKind.AssertRelation:
return(BuildAssert((BoundAssertRelation)relation));
case BoundNodeKind.TableSpoolPusher:
return(BuildTableSpoolPusher((BoundTableSpoolPusher)relation));
case BoundNodeKind.TableSpoolPopper:
return(BuildTableSpoolPopper((BoundTableSpoolPopper)relation));
default:
throw ExceptionBuilder.UnexpectedValue(relation.Kind);
}
}
public override BoundRelation RewriteRelation(BoundRelation node)
{
_subqueryStack.Push(new List <Subquery>());
var result = base.RewriteRelation(node);
if (_subqueryStack.Peek().Any())
{
throw ExceptionBuilder.UnexpectedValue(node.Kind);
}
_subqueryStack.Pop();
return(result);
}
private BoundRelation RewriteInputWithSubqueries(BoundRelation input)
{
var subqueries = _subqueryStack.Peek();
var result = input;
result = subqueries.Where(s => s.Kind == SubqueryKind.Exists)
.Aggregate(result, EmitExists);
result = subqueries.Where(s => s.Kind == SubqueryKind.Subselect)
.Aggregate(result, EmitSubselect);
subqueries.Clear();
return(result);
}
private bool IsInstantiatedCommonTableExpression(BoundRelation boundRelation)
{
if (_symbol == null)
{
return(false);
}
var tableRelation = boundRelation as BoundTableRelation;
if (tableRelation == null)
{
return(false);
}
return(tableRelation.TableInstance.Table == _symbol);
}
private static BoundRelation AssembleJoin(ICollection <JoinNode> nodes, ICollection <JoinEdge> edges)
{
var nodeComparer = Comparer <JoinNode> .Create(CompareNode);
var remainingNodes = new HashSet <JoinNode>(nodes);
var remainingEdges = new HashSet <JoinEdge>(edges);
var candidateNodes = new HashSet <JoinNode>();
BoundRelation result = null;
while (remainingNodes.Count > 0)
{
var start = remainingNodes.OrderBy(n => n, nodeComparer).First();
remainingNodes.Remove(start);
var relation = start.Relation;
candidateNodes.UnionWith(start.Edges.Select(e => e.Other(start)));
while (candidateNodes.Count > 0)
{
var usableEdges = candidateNodes.SelectMany(n => n.Edges)
.Where(e => remainingEdges.Contains(e))
.Where(n => !remainingNodes.Contains(n.Left) || !remainingNodes.Contains(n.Right))
.Where(e => e.Conditions.Any(IsRelation))
.ToImmutableArray();
if (!usableEdges.Any())
{
usableEdges = candidateNodes.SelectMany(n => n.Edges)
.Where(e => remainingEdges.Contains(e))
.Where(n => !remainingNodes.Contains(n.Left) || !remainingNodes.Contains(n.Right))
.ToImmutableArray();
}
var nextNode = usableEdges.SelectMany(e => new[] { e.Left, e.Right })
.Where(candidateNodes.Contains)
.OrderBy(n => n, nodeComparer)
.FirstOrDefault();
if (nextNode != null)
{
candidateNodes.Remove(nextNode);
candidateNodes.UnionWith(nextNode.Edges.Select(e => e.Other(nextNode)).Where(n => remainingNodes.Contains(n)));
var edge = usableEdges.First(e => !remainingNodes.Contains(e.Left) && e.Right == nextNode ||
!remainingNodes.Contains(e.Right) && e.Left == nextNode);
remainingNodes.Remove(nextNode);
remainingEdges.Remove(edge);
var left = relation;
var right = nextNode.Relation;
var condition = Expression.And(edge.Conditions);
relation = new BoundJoinRelation(BoundJoinType.Inner, left, right, condition, null, null);
}
}
result = result == null
? relation
: new BoundJoinRelation(BoundJoinType.Inner, result, relation, null, null, null);
}
Debug.Assert(remainingNodes.Count == 0, @"Found remaining nodes");
// Add filter for remaining predicates
var remainingPredicates = remainingEdges.SelectMany(e => e.Conditions);
var remainingCondition = Expression.And(remainingPredicates);
if (remainingCondition != null)
{
result = new BoundFilterRelation(result, remainingCondition);
}
return(result);
}
public static Iterator Build(BoundRelation relation)
{
var builder = new IteratorBuilder();
return(builder.BuildRelation(relation));
}
private static BoundJoinRelation EmitSubselect(BoundRelation result, Subquery subselect)
{
return(new BoundJoinRelation(BoundJoinType.LeftOuter, result, subselect.Relation, null, null, subselect.Passthru));
}
private static BoundJoinRelation EmitExists(BoundRelation result, Subquery existsSubquery)
{
return(new BoundJoinRelation(BoundJoinType.LeftSemi, result, existsSubquery.Relation, null, existsSubquery.ValueSlot, existsSubquery.Passthru));
}
private RowBufferAllocation BuildRowBufferAllocation(BoundRelation input, RowBuffer rowBuffer)
{
return(new RowBufferAllocation(_outerRowBufferAllocation, rowBuffer, input.GetOutputValues()));
}
public override BoundRelation RewriteRelation(BoundRelation node)
{
AddValueSlotMapping(node.GetDefinedValues());
return(base.RewriteRelation(node));
}
public static BoundRelation Instantiate(BoundRelation relation, IEnumerable <KeyValuePair <ValueSlot, ValueSlot> > mapping)
{
var valueSlotRewriter = new ValueSlotRewriter(mapping);
return(valueSlotRewriter.RewriteRelation(relation));
}
public static BoundRelation Instantiate(BoundRelation relation)
{
return(Instantiate(relation, Enumerable.Empty <KeyValuePair <ValueSlot, ValueSlot> >()));
}
private BoundRelation InstantiateRecursiveCommonTableExpression(ImmutableArray <ValueSlot> outputValues, CommonTableExpressionSymbol symbol)
{
// TableSpoolPusher
// Concat
// Compute (Recursion := 0)
// <Anchor>
// Assert (Recursion <= 100)
// Inner Join
// Compute (Recursion := Recursion + 1)
// TableSpoolPopper
// Concat
// Filter <RecursiveJoinPredicate1>
// <RecursiveMember1>
// Filter <RecursiveJoinPredicate2>
// <RecursiveMember2>
var valueSlotFactory = outputValues.First().Factory;
// Create output values
var unionRecusionSlot = valueSlotFactory.CreateTemporary(typeof(int));
var concatValueSlots = outputValues.Add(unionRecusionSlot);
// Create anchor
var anchor = RewriteRelation(Instatiator.Instantiate(symbol.Anchor.Relation));
var anchorValues = anchor.GetOutputValues().ToImmutableArray();
var initRecursionSlot = valueSlotFactory.CreateTemporary(typeof(int));
var initRecursionDefinition = new BoundComputedValue(Expression.Literal(0), initRecursionSlot);
var initRecursion = new BoundComputeRelation(anchor, ImmutableArray.Create(initRecursionDefinition));
var initRecursionOutputs = anchorValues.Add(initRecursionSlot);
// Create TableSpoolPopper
var tableSpoolPopperSlots = initRecursionOutputs.Select(v => v.Duplicate()).ToImmutableArray();
var tableSpoolPopper = new BoundTableSpoolPopper(tableSpoolPopperSlots);
var anchorRecursionCounter = tableSpoolPopperSlots.Last();
var inc = Expression.Plus(Expression.Value(anchorRecursionCounter), Expression.Literal(1));
var incRecursionSlot = valueSlotFactory.CreateTemporary(typeof(int));
var incRecursionDefinition = new BoundComputedValue(inc, incRecursionSlot);
var incRecursion = new BoundComputeRelation(tableSpoolPopper, ImmutableArray.Create(incRecursionDefinition));
// Create recursive members
var recursiveRewriter = new CommonTableExpressionInstantiator(symbol);
var recursiveMembers = new List <BoundRelation>(symbol.RecursiveMembers.Length);
var recursiveMemberOutputs = new List <ImmutableArray <ValueSlot> >(symbol.RecursiveMembers.Length);
var anchorReferences = tableSpoolPopperSlots.RemoveAt(tableSpoolPopperSlots.Length - 1);
foreach (var recursiveMember in symbol.RecursiveMembers)
{
var recursivePrototype = recursiveMember.Relation;
var mapping = CreateRecursiveMemberInstanceValueSlotMapping(symbol, anchorReferences, recursivePrototype);
var recursiveInstance = Instatiator.Instantiate(recursivePrototype, mapping);
var recursiveRelation = recursiveRewriter.RewriteRelation(recursiveInstance);
var recursiveRelationOutputs = recursiveRelation.GetOutputValues().ToImmutableArray();
recursiveMembers.Add(recursiveRelation);
recursiveMemberOutputs.Add(recursiveRelationOutputs);
}
// Concatenate recursive members
var recursiveConcatValues = Enumerable
.Range(0, concatValueSlots.Length - 1)
.Select(i =>
{
var slot = valueSlotFactory.CreateTemporary(concatValueSlots[i].Type);
return(new BoundUnifiedValue(slot, recursiveMemberOutputs.Select(o => o[i])));
})
.ToImmutableArray();
var hasSingleRecursiveMember = recursiveMembers.Count == 1;
var recursiveConcat = hasSingleRecursiveMember ? recursiveMembers.Single() : new BoundConcatenationRelation(recursiveMembers, recursiveConcatValues);
var recursionOutputs = hasSingleRecursiveMember ? recursiveMemberOutputs.Single() : recursiveConcatValues.Select(u => u.ValueSlot).ToImmutableArray();
// Create inner join
var join = new BoundJoinRelation(BoundJoinType.Inner, incRecursion, recursiveConcat, null, null, null);
var joinOutputs = recursionOutputs.Add(incRecursionSlot);
var recursiveProjection = new BoundProjectRelation(join, joinOutputs);
// Create assert
var assertCondition = Expression.LessThan(Expression.Value(incRecursionSlot), Expression.Literal(100));
var assert = new BoundAssertRelation(recursiveProjection, assertCondition, Resources.MaximumRecursionLevelExceeded);
// Create top level concat
var concatValues = concatValueSlots.Select((v, i) =>
{
var slots = new[]
{
initRecursionOutputs[i],
joinOutputs[i]
};
return(new BoundUnifiedValue(v, slots));
});
var concatInputs = new BoundRelation[] { initRecursion, assert };
var concat = new BoundConcatenationRelation(concatInputs, concatValues);
var tableSpoolPusher = new BoundTableSpoolPusher(concat);
return(new BoundProjectRelation(tableSpoolPusher, concatValueSlots.Take(concatValueSlots.Length - 1)));
}
public JoinNode(BoundRelation relation)
{
Relation = relation;
}
private static ImmutableDictionary <ValueSlot, ValueSlot> CreateRecursiveMemberInstanceValueSlotMapping(CommonTableExpressionSymbol symbol, ImmutableArray <ValueSlot> instanceSlots, BoundRelation relation)
{
var finder = new CommonTableExpressionInstanceFinder(symbol);
finder.VisitRelation(relation);
Debug.Assert(finder.Instance != null && finder.Instance.Table == symbol);
Debug.Assert(instanceSlots.Length == finder.Instance.ColumnInstances.Length);
return(instanceSlots.Zip(finder.Instance.ColumnInstances, (v, c) => new KeyValuePair <ValueSlot, ValueSlot>(c.ValueSlot, v))
.ToImmutableDictionary());
}
private static BoundRelation WrapWithFilter(BoundRelation input, BoundExpression predicate)
{
return(predicate == null
? input
: new BoundFilterRelation(input, predicate));
}
public static CardinalityEstimate Estimate(BoundRelation relation)
{
switch (relation.Kind)
{
case BoundNodeKind.TableRelation:
return(EstimateTableRelation((BoundTableRelation)relation));
case BoundNodeKind.DerivedTableRelation:
return(EstimateDerivedTableRelation((BoundDerivedTableRelation)relation));
case BoundNodeKind.JoinRelation:
return(EstimateJoinRelation((BoundJoinRelation)relation));
case BoundNodeKind.HashMatchRelation:
return(EstimateHashMatchRelation((BoundHashMatchRelation)relation));
case BoundNodeKind.ComputeRelation:
return(EstimateComputeRelation((BoundComputeRelation)relation));
case BoundNodeKind.FilterRelation:
return(EstimateFilterRelation((BoundFilterRelation)relation));
case BoundNodeKind.GroupByAndAggregationRelation:
return(EstimateGroupByAndAggregationRelation((BoundGroupByAndAggregationRelation)relation));
case BoundNodeKind.StreamAggregatesRelation:
return(EstimateStreamAggregatesRelation((BoundStreamAggregatesRelation)relation));
case BoundNodeKind.ConstantRelation:
return(EstimateConstantRelation((BoundConstantRelation)relation));
case BoundNodeKind.UnionRelation:
return(EstimateUnionRelation((BoundUnionRelation)relation));
case BoundNodeKind.ConcatenationRelation:
return(EstimateConcatenationRelation((BoundConcatenationRelation)relation));
case BoundNodeKind.IntersectOrExceptRelation:
return(EstimateIntersectOrExceptRelation((BoundIntersectOrExceptRelation)relation));
case BoundNodeKind.ProjectRelation:
return(EstimateProjectRelation((BoundProjectRelation)relation));
case BoundNodeKind.SortRelation:
return(EstimateSortRelation((BoundSortRelation)relation));
case BoundNodeKind.TopRelation:
return(EstimateTopRelation((BoundTopRelation)relation));
case BoundNodeKind.AssertRelation:
return(EstimateAssertRelation((BoundAssertRelation)relation));
case BoundNodeKind.TableSpoolPusher:
return(EstimateTableSpoolPusher((BoundTableSpoolPusher)relation));
case BoundNodeKind.TableSpoolPopper:
return(EstimateTableSpoolPopper((BoundTableSpoolPopper)relation));
default:
throw ExceptionBuilder.UnexpectedValue(relation.Kind);
}
}
