/// <summary>
/// Deep Copy
/// </summary>
/// <param name="component"></param>
public MatchComponent(MatchComponent component)
{
Nodes = component.Nodes;
EdgeMaterilizedDict = new Dictionary <MatchEdge, bool>(component.EdgeMaterilizedDict);
MaterializedNodeSplitCount = new Dictionary <MatchNode, int>(component.MaterializedNodeSplitCount);
UnmaterializedNodeMapping = new Dictionary <MatchNode, List <MatchEdge> >();
foreach (var nodeMapping in component.UnmaterializedNodeMapping)
{
UnmaterializedNodeMapping[nodeMapping.Key] = new List <MatchEdge>(nodeMapping.Value);
}
this.TraversalChain = new List <Tuple <MatchNode, MatchEdge, MatchNode, List <MatchEdge>, List <MatchEdge> > >();
foreach (var chain in component.TraversalChain)
{
this.TraversalChain.Add(new Tuple <MatchNode, MatchEdge, MatchNode, List <MatchEdge>, List <MatchEdge> >(
chain.Item1, chain.Item2, chain.Item3, chain.Item4, chain.Item5));
}
NodeToMaterializedEdgesDict = new Dictionary <string, List <Tuple <MatchEdge, MaterializedEdgeType> > >();
foreach (var nodeMatEdges in component.NodeToMaterializedEdgesDict)
{
NodeToMaterializedEdgesDict[nodeMatEdges.Key] = new List <Tuple <MatchEdge, MaterializedEdgeType> >(nodeMatEdges.Value);
}
SinkNodeStatisticsDict = new Dictionary <MatchNode, Statistics>();
foreach (var stat in component.SinkNodeStatisticsDict)
{
SinkNodeStatisticsDict[stat.Key] = stat.Value;
}
Cardinality = component.Cardinality;
Cost = component.Cost;
}
public override List <Tuple <MatchNode, MatchEdge, List <MatchEdge>, List <MatchEdge>, List <MatchEdge> > > GetOptimizedTraversalOrder(ConnectedComponent subGraph)
{
if (subGraph.Nodes.Count == 1)
{
return
(this.GenerateTraversalOrderFromTraversalChain(
new List <Tuple <MatchNode, MatchEdge, MatchNode, List <MatchEdge>, List <MatchEdge> > >
{
new Tuple <MatchNode, MatchEdge, MatchNode, List <MatchEdge>, List <MatchEdge> >(
subGraph.Nodes.First().Value, null, null, null, null)
}));
}
// If it exists, pick a node without incoming edges as the start point
List <MatchComponent> componentStates = subGraph.Nodes.Where(node => node.Value.ReverseNeighbors.Count == 0).
Select(node => new MatchComponent(node.Value)).Take(1).ToList();
// Otherwise, pick a node randomly as the start point
if (!componentStates.Any())
{
componentStates.Add(new MatchComponent(subGraph.Nodes.First().Value));
}
// DP
while (componentStates.Any())
{
List <MatchComponent> nextCompnentStates = new List <MatchComponent>();
// Iterate on current components
foreach (MatchComponent curComponent in componentStates)
{
OneHeightTree nodeUnits = this.GetNodeUnits(subGraph, curComponent);
if (nodeUnits == null &&
curComponent.ActiveNodeCount == subGraph.ActiveNodeCount &&
curComponent.EdgeMaterilizedDict.Count(e => e.Value == true) == subGraph.Edges.Count(e => e.Value.IsDanglingEdge == false))
{
return(this.GenerateTraversalOrderFromTraversalChain(curComponent.TraversalChain));
}
CandidateJoinUnit candidateUnit = this.GetCandidateUnits2(nodeUnits, curComponent);
// Add it to the current component to generate next states
MatchComponent newComponent = this.GetNextState(curComponent, candidateUnit);
if (nextCompnentStates.Count >= MaxStates)
{
throw new GraphViewException("This graph pattern is not supported yet.");
}
nextCompnentStates.Add(newComponent);
}
componentStates = nextCompnentStates;
}
return(null);
}
/// <summary>
/// Get a full one height tree with joint edges and unmaterlized edges.
/// </summary>
/// <param name="graph"></param>
/// <param name="component"></param>
/// <returns></returns>
private OneHeightTree GetNodeUnits(ConnectedComponent graph, MatchComponent component)
{
var useOriginalEdge = new List <OneHeightTree>();
var useRevEdge = new List <OneHeightTree>();
// Return node deinfed in the outer context preferably
foreach (var node in graph.Nodes.Values.Where(n => n.IsFromOuterContext && !component.Nodes.Contains(n)))
{
var remainingEdges = node.Neighbors.Where(e => !component.EdgeMaterilizedDict.ContainsKey(e)).ToList();
if (component.UnmaterializedNodeMapping.ContainsKey(node) ||
remainingEdges.Any(e => component.Nodes.Contains(e.SinkNode)))
{
return(new OneHeightTree
{
TreeRoot = node,
Edges = remainingEdges
});
}
}
foreach (var node in graph.Nodes.Values.Where(n => !component.Nodes.Contains(n)))
{
var remainingEdges = node.Neighbors.Where(e => !component.EdgeMaterilizedDict.ContainsKey(e)).ToList();
if (component.UnmaterializedNodeMapping.ContainsKey(node))
{
useOriginalEdge.Add(new OneHeightTree
{
TreeRoot = node,
Edges = remainingEdges,
});
break;
}
if (remainingEdges.Any(e => component.Nodes.Contains(e.SinkNode)))
{
useRevEdge.Add(new OneHeightTree
{
TreeRoot = node,
Edges = remainingEdges
});
}
}
if (useOriginalEdge.Any())
{
return(useOriginalEdge[0]);
}
if (useRevEdge.Any())
{
return(useRevEdge[0]);
}
return(null);
}
internal MatchComponent GetNextState2(MatchComponent curComponent, CandidateJoinUnit candidateTree)
{
// Deep copy the component
var newComponent = new MatchComponent(curComponent);
// Update component
UpdateComponent(newComponent, candidateTree);
// Construct traversal chain and Update join cost
ConstructTraversalChainAndUpdateCost2(newComponent, candidateTree);
return(newComponent);
}
private void ConstructTraversalChainAndUpdateCost2(MatchComponent curComponent, CandidateJoinUnit nodeUnitCandidate)
{
var inPreMatEdges = nodeUnitCandidate.PreMatIncomingEdges;
var inPostMatEdges = nodeUnitCandidate.PostMatIncomingEdges;
var outPostMatEdges = nodeUnitCandidate.PostMatOutgoingEdges;
curComponent.TraversalChain2.Add(
new Tuple <MatchNode, MatchEdge, MatchNode, List <MatchEdge>, List <MatchEdge> >(
inPreMatEdges[0].SourceNode,
inPreMatEdges[0],
inPreMatEdges[0].SinkNode,
outPostMatEdges,
inPostMatEdges));
}
/// <summary>
/// Get a full one height tree with joint edges and unmaterlized edges.
/// </summary>
/// <param name="graph"></param>
/// <param name="component"></param>
/// <returns></returns>
private OneHeightTree GetNodeUnits(ConnectedComponent graph, MatchComponent component)
{
foreach (MatchNode node in graph.Nodes.Values.Where(n => !component.Nodes.ContainsKey(n.NodeAlias)))
{
List <MatchEdge> remainingEdges = node.Neighbors.Where(e => !component.EdgeMaterilizedDict.ContainsKey(e)).ToList();
if (component.UnmaterializedNodeMapping.ContainsKey(node) ||
remainingEdges.Any(e => component.Nodes.ContainsKey(e.SinkNode.NodeAlias)))
{
return(new OneHeightTree
{
TreeRoot = node,
Edges = remainingEdges,
});
}
}
return(null);
}
/// <summary>
/// Transit from current component to the new component in the next state given the Node Unit
/// </summary>
/// <param name="candidateTree"></param>
/// <param name="statisticsCalculator"></param>
/// <returns></returns>
public MatchComponent GetNextState(
CandidateJoinUnit candidateTree,
IMatchJoinStatisticsCalculator statisticsCalculator,
GraphMetaData metaData)
{
// Deep copy the component
var newComponent = new MatchComponent(this);
// Constrcuts join conditions and retrieves join selectivity
double joinSelectivity;
double sqlEstimatedJoinSelectivity;
var joinCondition = newComponent.ConstructJoinCondition(candidateTree, statisticsCalculator, metaData, out joinSelectivity,
out sqlEstimatedJoinSelectivity);
// Constructs physical join method and join table references
newComponent.ConstructPhysicalJoinAndUpdateCost(candidateTree, joinCondition,
joinSelectivity, sqlEstimatedJoinSelectivity, metaData);
return(newComponent);
}
/// <summary>
/// Calculate the number used for adjusting the SQL Server estimation in the downsize function.
/// </summary>
/// <param name="component"></param>
/// <param name="joinCondition"></param>
private static WTableReference AdjustEstimation(MatchComponent component, out WBooleanExpression joinCondition, out double affectedSqlEstimatedSize)
{
const int sizeFactor = 10;
int estimateFactor = 0;
double size = component.Cardinality;
double estimatedSize = component.SqlEstimatedSize;
double shrinkSize = component.RightestTableRefSize;
WTableReference tableReference = component.TableRef;
affectedSqlEstimatedSize = 1.0;
joinCondition = null;
if (size > sizeFactor * estimatedSize)
{
estimateFactor = (int)Math.Ceiling(size / estimatedSize);
}
else if (sizeFactor * size < estimatedSize)
{
shrinkSize = 1.0 / (1 - Math.Pow((1 - 1.0 / shrinkSize), 1.5));
if (estimatedSize < shrinkSize)
{
affectedSqlEstimatedSize /= estimatedSize;
estimatedSize = 1;
}
else
{
affectedSqlEstimatedSize /= shrinkSize;
estimatedSize /= shrinkSize;
}
estimateFactor = (int)Math.Ceiling(size / estimatedSize);
joinCondition = ConstructDownSizeJoinCondition(component.RightestTableAlias);
}
if (estimateFactor > 1)
{
double affectedUpSize;
tableReference = ConstructUpSizeTableReference(tableReference, estimateFactor,
out affectedUpSize);
affectedSqlEstimatedSize *= affectedUpSize;
}
return(tableReference);
}
private void ConstructTraversalChainAndUpdateCost(MatchComponent curComponent, CandidateJoinUnit nodeUnitCandidate)
{
List <MatchEdge> inPreMatEdges = nodeUnitCandidate.PreMatIncomingEdges;
List <MatchEdge> inPostMatEdges = nodeUnitCandidate.PostMatIncomingEdges;
List <MatchEdge> outPostMatEdges = nodeUnitCandidate.PostMatOutgoingEdges;
//
// Item1: sourceNode
// Item2: traversalEdge
// Item3: sinkNode
// Item4: backwardingEdges
// Item5: forwardingEdges
//
curComponent.TraversalChain.Add(
new Tuple <MatchNode, MatchEdge, MatchNode, List <MatchEdge>, List <MatchEdge> >(
curComponent.Nodes[inPreMatEdges[0].SourceNode.NodeAlias],
inPreMatEdges[0],
curComponent.Nodes[inPreMatEdges[0].SinkNode.NodeAlias],
outPostMatEdges,
inPostMatEdges));
}
/// <summary>
/// Deep Copy
/// </summary>
/// <param name="component"></param>
public MatchComponent(MatchComponent component)
{
Nodes = new HashSet <MatchNode>(component.Nodes);
EdgeMaterilizedDict = new Dictionary <MatchEdge, bool>(component.EdgeMaterilizedDict);
MaterializedNodeSplitCount = new Dictionary <MatchNode, int>(component.MaterializedNodeSplitCount);
UnmaterializedNodeMapping = new Dictionary <MatchNode, List <MatchEdge> >();
foreach (var nodeMapping in component.UnmaterializedNodeMapping)
{
UnmaterializedNodeMapping[nodeMapping.Key] = new List <MatchEdge>(nodeMapping.Value);
}
SinkNodeStatisticsDict = new Dictionary <MatchNode, Statistics>(component.SinkNodeStatisticsDict);
TableRef = component.TableRef;
Cardinality = component.Cardinality;
Cost = component.Cost;
DeltaMemory = component.DeltaMemory;
TotalMemory = component.TotalMemory;
SqlEstimatedDeltaMemory = component.SqlEstimatedDeltaMemory;
SqlEstimatedTotalMemory = component.SqlEstimatedTotalMemory;
SqlEstimatedSize = component.SqlEstimatedSize;
RightestTableAlias = component.RightestTableAlias;
RightestTableRefSize = component.RightestTableRefSize;
}
/// <summary>
/// Deep Copy
/// </summary>
/// <param name="component"></param>
public MatchComponent(MatchComponent component)
{
Nodes = new List <MatchNode>(component.Nodes);
EdgeMaterilizedDict = new Dictionary <MatchEdge, bool>(component.EdgeMaterilizedDict);
MaterializedNodeSplitCount = new Dictionary <MatchNode, int>(component.MaterializedNodeSplitCount);
UnmaterializedNodeMapping = new Dictionary <MatchNode, List <MatchEdge> >();
foreach (var nodeMapping in component.UnmaterializedNodeMapping)
{
UnmaterializedNodeMapping[nodeMapping.Key] = new List <MatchEdge>(nodeMapping.Value);
}
StatisticsDict = new Dictionary <MatchNode, ColumnStatistics>(component.StatisticsDict);
TableRef = component.TableRef;
Size = component.Size;
Cost = component.Cost;
DeltaMemory = component.DeltaMemory;
TotalMemory = component.TotalMemory;
EstimateDeltaMemory = component.EstimateDeltaMemory;
EstimateTotalMemory = component.EstimateTotalMemory;
EstimateSize = component.EstimateSize;
FatherOfRightestTableRef = component.FatherOfRightestTableRef;
RightestTableRefSize = component.RightestTableRefSize;
FatherListofDownSizeTable = new List <Tuple <WQualifiedJoin, String> >(component.FatherListofDownSizeTable);
Context = component.Context;
}
private CandidateJoinUnit GetCandidateUnits2(OneHeightTree tree, MatchComponent component)
{
var revEdgeDict = Graph.ReversedEdgeDict;
var root = tree.TreeRoot;
List <MatchEdge> inEdges;
component.UnmaterializedNodeMapping.TryGetValue(root, out inEdges);
var outEdges = new List <MatchEdge>();
var unpopEdges = new List <MatchEdge>();
foreach (var edge in tree.Edges)
{
if (component.Nodes.Contains(edge.SinkNode))
{
outEdges.Add(edge);
}
else
{
unpopEdges.Add(edge);
}
}
var rawEdges = new Dictionary <string, Tuple <MatchEdge, EdgeDir> >();
var extInEdges = new Dictionary <string, MatchEdge>();
if (inEdges != null)
{
rawEdges = inEdges.ToDictionary(edge => edge.EdgeAlias,
edge => new Tuple <MatchEdge, EdgeDir>(edge, EdgeDir.In));
extInEdges = inEdges.ToDictionary(edge => edge.EdgeAlias);
}
foreach (var edge in outEdges)
{
var key = edge.EdgeAlias;
rawEdges.Add(key, new Tuple <MatchEdge, EdgeDir>(edge, EdgeDir.Out));
extInEdges.Add(key, revEdgeDict[key]);
}
if (extInEdges.Any())
{
var firstEdge = extInEdges.FirstOrDefault(e => e.Value.IsReversed == false);
if (firstEdge.Value == null)
{
firstEdge = extInEdges.First();
}
var preMatInEdges = new Dictionary <string, MatchEdge>
{
{ firstEdge.Key, firstEdge.Value }
};
var postMatEdges = rawEdges.Where(entry => !preMatInEdges.ContainsKey(entry.Key))
.Select(entry => entry.Value).ToList();
// Both edge will be forced to choose the incoming direction type
var postMatIncomingEdges = postMatEdges.Where(entry => entry.Item2 == EdgeDir.In || (entry.Item2 == EdgeDir.Out && entry.Item1.EdgeType == WEdgeType.BothEdge))
.Select(entry => (entry.Item2 == EdgeDir.In ? entry.Item1 : revEdgeDict[entry.Item1.EdgeAlias])).ToList();
var postMatOutgoingEdges = postMatEdges.Where(entry => entry.Item2 == EdgeDir.Out && entry.Item1.EdgeType != WEdgeType.BothEdge)
.Select(entry => entry.Item1).ToList();
return(new CandidateJoinUnit
{
TreeRoot = root,
PreMatIncomingEdges = preMatInEdges.Select(entry => entry.Value).ToList(),
PreMatOutgoingEdges = new List <MatchEdge>(),
PostMatIncomingEdges = postMatIncomingEdges,
PostMatOutgoingEdges = postMatOutgoingEdges,
UnmaterializedEdges = unpopEdges,
});
}
else
{
throw new GraphViewException("This graph pattern is not yet supported.");
}
}
/// <summary>
/// Calculate the number used for adjusting the SQL Server estimation in the downsize function.
/// </summary>
/// <param name="component"></param>
/// <param name="matEdges"></param>
/// <param name="joinCondition"></param>
private static WTableReference AdjustEstimation(MatchComponent component, List<MatchEdge> matEdges, out WBooleanExpression joinCondition, out double affectedSqlEstimatedSize)
{
const int sizeFactor = 10;
int estimateFactor = 0;
double size = component.Cardinality*
matEdges.Select(e => e.AverageDegree).Aggregate(1.0, (cur, next) => cur*next);
double estimatedSize = component.SqlEstimatedSize * Math.Pow(1000, matEdges.Count);
double shrinkSize = component.RightestTableRefSize;
WTableReference tableReference = component.TableRef;
affectedSqlEstimatedSize = 1.0;
joinCondition = null;
if (size > sizeFactor*estimatedSize)
{
estimateFactor = (int)Math.Ceiling(size / estimatedSize);
}
else if (sizeFactor*size < estimatedSize)
{
shrinkSize = 1.0/(1 - Math.Pow((1 - 1.0/shrinkSize), 1.5));
if (estimatedSize < shrinkSize)
{
affectedSqlEstimatedSize /= estimatedSize;
estimatedSize = 1;
}
else
{
affectedSqlEstimatedSize /= shrinkSize;
estimatedSize /= shrinkSize;
}
estimateFactor = (int) Math.Ceiling(size/estimatedSize);
joinCondition = ConstructDownSizeJoinCondition(component.RightestTableAlias);
}
if (estimateFactor > 1)
{
double affectedUpSize;
tableReference = ConstructUpSizeTableReference(tableReference, estimateFactor,
out affectedUpSize);
affectedSqlEstimatedSize *= affectedUpSize;
}
return tableReference;
}
/// <summary>
/// Deep Copy
/// </summary>
/// <param name="component"></param>
public MatchComponent(MatchComponent component)
{
Nodes = new HashSet<MatchNode>(component.Nodes);
EdgeMaterilizedDict = new Dictionary<MatchEdge, bool>(component.EdgeMaterilizedDict);
MaterializedNodeSplitCount = new Dictionary<MatchNode, int>(component.MaterializedNodeSplitCount);
UnmaterializedNodeMapping = new Dictionary<MatchNode, List<MatchEdge>>();
foreach (var nodeMapping in component.UnmaterializedNodeMapping)
{
UnmaterializedNodeMapping[nodeMapping.Key] = new List<MatchEdge>(nodeMapping.Value);
}
SinkNodeStatisticsDict = new Dictionary<MatchNode, Statistics>(component.SinkNodeStatisticsDict);
TableRef = component.TableRef;
LastPostMatEdgeAlias = component.LastPostMatEdgeAlias;
LastJoinPostMatEdgesCount = component.LastJoinPostMatEdgesCount;
LastJoinSqlEstCardinality = component.LastJoinSqlEstCardinality;
LastJoinHint = component.LastJoinHint;
LastTable = component.LastTable;
Cardinality = component.Cardinality;
Cost = component.Cost;
DeltaMemory = component.DeltaMemory;
TotalMemory = component.TotalMemory;
SqlEstimatedDeltaMemory = component.SqlEstimatedDeltaMemory;
SqlEstimatedTotalMemory = component.SqlEstimatedTotalMemory;
SqlEstimatedSize = component.SqlEstimatedSize;
RightestTableAlias = component.RightestTableAlias;
RightestTableRefSize = component.RightestTableRefSize;
LastTableAlias = component.LastTableAlias;
WhereCondition = ObjectExtensions.Copy(component.WhereCondition);
}
/// <summary>
/// Calculate the number used for adjusting the SQL Server estimation in the downsize function.
/// </summary>
/// <param name="component"></param>
/// <param name="joinCondition"></param>
private static WTableReference AdjustEstimation(MatchComponent component, out WBooleanExpression joinCondition, out double affectedSqlEstimatedSize)
{
const int sizeFactor = 10;
int estimateFactor = 0;
double size = component.Cardinality;
double estimatedSize = component.SqlEstimatedSize;
double shrinkSize = component.RightestTableRefSize;
WTableReference tableReference = component.TableRef;
affectedSqlEstimatedSize = 1.0;
joinCondition = null;
if (size > sizeFactor*estimatedSize)
{
estimateFactor = (int)Math.Ceiling(size / estimatedSize);
}
else if (sizeFactor*size < estimatedSize)
{
shrinkSize = 1.0/(1 - Math.Pow((1 - 1.0/shrinkSize), 1.5));
if (estimatedSize < shrinkSize)
{
affectedSqlEstimatedSize /= estimatedSize;
estimatedSize = 1;
}
else
{
affectedSqlEstimatedSize /= shrinkSize;
estimatedSize /= shrinkSize;
}
estimateFactor = (int) Math.Ceiling(size/estimatedSize);
joinCondition = ConstructDownSizeJoinCondition(component.RightestTableAlias);
}
if (estimateFactor > 1)
{
double affectedUpSize;
tableReference = ConstructUpSizeTableReference(tableReference, estimateFactor,
out affectedUpSize);
affectedSqlEstimatedSize *= affectedUpSize;
}
return tableReference;
}
private void ConstructTraversalChainAndUpdateCost(MatchComponent curComponent, CandidateJoinUnit nodeUnitCandidate)
{
var inPreMatEdges = nodeUnitCandidate.PreMatIncomingEdges;
curComponent.TraversalChain.Add(new Tuple <MatchNode, MatchEdge>(inPreMatEdges[0].SourceNode, inPreMatEdges[0]));
}
/// <summary>
/// Deep Copy
/// </summary>
/// <param name="component"></param>
public MatchComponent(MatchComponent component)
{
Nodes = new List<MatchNode>(component.Nodes);
EdgeMaterilizedDict = new Dictionary<MatchEdge, bool>(component.EdgeMaterilizedDict);
MaterializedNodeSplitCount = new Dictionary<MatchNode, int>(component.MaterializedNodeSplitCount);
UnmaterializedNodeMapping = new Dictionary<MatchNode, List<MatchEdge>>();
foreach (var nodeMapping in component.UnmaterializedNodeMapping)
{
UnmaterializedNodeMapping[nodeMapping.Key] = new List<MatchEdge>(nodeMapping.Value);
}
StatisticsDict = new Dictionary<MatchNode, ColumnStatistics>(component.StatisticsDict);
TableRef = component.TableRef;
Size = component.Size;
Cost = component.Cost;
DeltaMemory = component.DeltaMemory;
TotalMemory = component.TotalMemory;
EstimateDeltaMemory = component.EstimateDeltaMemory;
EstimateTotalMemory = component.EstimateTotalMemory;
EstimateSize = component.EstimateSize;
FatherOfRightestTableRef = component.FatherOfRightestTableRef;
RightestTableRefSize = component.RightestTableRefSize;
FatherListofDownSizeTable = new List<Tuple<WQualifiedJoin, String>>(component.FatherListofDownSizeTable);
Context = component.Context;
}
/// <summary>
/// Get a full one height tree with joint edges and unmaterlized edges,
/// returns a tuple whose first item is the one height tree and the second item
/// indicates whether the one height tree only joins with the component's materialized
/// node on its root.
/// </summary>
/// <param name="graph"></param>
/// <param name="component"></param>
/// <returns></returns>
public IEnumerable<Tuple<OneHeightTree,bool>> GetNodeUnits(ConnectedComponent graph, MatchComponent component)
{
var nodes = graph.Nodes;
foreach (var node in nodes.Values.Where(e => !graph.IsTailNode[e]))
{
//var newJoinUnitList = new List<MatchJoinUnit>()
bool joint = false;
var jointEdges = new List<MatchEdge>();
var nodeEdgeDict = node.Neighbors.ToDictionary(e => e,
e => component.EdgeMaterilizedDict.ContainsKey(e));
// Edge to component node
foreach (var edge in node.Neighbors.Where(e => !nodeEdgeDict[e]))
{
if (component.Nodes.Contains(edge.SinkNode))
{
joint = true;
nodeEdgeDict[edge] = true;
jointEdges.Add(edge);
}
}
// Component edge to node
if (!joint && component.UnmaterializedNodeMapping.ContainsKey(node))
{
joint = true;
}
// Add unpopulated edges
var nodeUnpopulatedEdges = nodeEdgeDict.Where(e => !e.Value).Select(e => e.Key).ToList();
if (joint)
yield return new Tuple<OneHeightTree, bool>(new OneHeightTree
{
TreeRoot = node,
MaterializedEdges = jointEdges,
UnmaterializedEdges = nodeUnpopulatedEdges,
}, false);
// Single node edge
else if (nodeUnpopulatedEdges.Count > 0 && component.MaterializedNodeSplitCount.Count > 1 && component.MaterializedNodeSplitCount.ContainsKey(node))
{
yield return new Tuple<OneHeightTree, bool>(new OneHeightTree
{
TreeRoot = node,
MaterializedEdges = jointEdges,
UnmaterializedEdges = nodeUnpopulatedEdges,
}, true);
}
}
}
private CandidateJoinUnit GetCandidateUnits2(OneHeightTree tree, MatchComponent component)
{
Dictionary <string, MatchEdge> revEdgeDict = this.Graph.ReversedEdgeDict;
MatchNode root = tree.TreeRoot;
List <MatchEdge> inEdges;
component.UnmaterializedNodeMapping.TryGetValue(root, out inEdges);
List <MatchEdge> outEdges = new List <MatchEdge>();
List <MatchEdge> unpopEdges = new List <MatchEdge>();
foreach (MatchEdge edge in tree.Edges)
{
if (component.Nodes.ContainsKey(edge.SinkNode.NodeAlias))
{
outEdges.Add(edge);
}
else
{
unpopEdges.Add(edge);
}
}
Dictionary <string, Tuple <MatchEdge, EdgeDir> > rawEdges = new Dictionary <string, Tuple <MatchEdge, EdgeDir> >();
Dictionary <string, MatchEdge> extInEdges = new Dictionary <string, MatchEdge>();
if (inEdges != null)
{
rawEdges = inEdges.ToDictionary(edge => edge.EdgeAlias,
edge => new Tuple <MatchEdge, EdgeDir>(edge, EdgeDir.In));
extInEdges = inEdges.ToDictionary(edge => edge.EdgeAlias);
}
foreach (MatchEdge edge in outEdges)
{
string key = edge.EdgeAlias;
rawEdges.Add(key, new Tuple <MatchEdge, EdgeDir>(edge, EdgeDir.Out));
extInEdges.Add(key, revEdgeDict[key]);
}
if (extInEdges.Any())
{
KeyValuePair <string, MatchEdge> firstEdge = extInEdges.FirstOrDefault(e => e.Value.IsReversed == false);
if (firstEdge.Value == null)
{
firstEdge = extInEdges.First();
}
Dictionary <string, MatchEdge> preMatInEdges = new Dictionary <string, MatchEdge>
{
{ firstEdge.Key, firstEdge.Value }
};
List <Tuple <MatchEdge, EdgeDir> > postMatEdges = rawEdges.Where(entry => !preMatInEdges.ContainsKey(entry.Key))
.Select(entry => entry.Value).ToList();
List <MatchEdge> postMatIncomingEdges =
postMatEdges.Where(entry => entry.Item2 == EdgeDir.In).Select(entry => entry.Item1).ToList();
List <MatchEdge> postMatOutgoingEdges =
postMatEdges.Where(entry => entry.Item2 == EdgeDir.Out).Select(entry => entry.Item1).ToList();
return(new CandidateJoinUnit
{
TreeRoot = root,
PreMatIncomingEdges = preMatInEdges.Select(entry => entry.Value).ToList(),
PreMatOutgoingEdges = new List <MatchEdge>(),
PostMatIncomingEdges = postMatIncomingEdges,
PostMatOutgoingEdges = postMatOutgoingEdges,
UnmaterializedEdges = unpopEdges,
});
}
else
{
throw new GraphViewException("This graph pattern is not yet supported.");
}
}
private void UpdateComponent(MatchComponent curComponent, CandidateJoinUnit candidateTree)
{
var nodes = curComponent.Nodes;
var edgeMaterializedDict = curComponent.EdgeMaterilizedDict;
var unmaterializedNodeMapping = curComponent.UnmaterializedNodeMapping;
var root = candidateTree.TreeRoot;
if (!nodes.Contains(root))
{
nodes.Add(root);
}
curComponent.MaterializedNodeSplitCount[root] = 0;
var inEdges =
candidateTree.PreMatIncomingEdges.Select(
e => new Tuple <MaterializedOrder, MatchEdge>(MaterializedOrder.Pre, e))
.Union(
candidateTree.PostMatIncomingEdges.Select(
e => new Tuple <MaterializedOrder, MatchEdge>(MaterializedOrder.Post, e)))
.ToList();
var outEdges =
candidateTree.PreMatOutgoingEdges.Select(
e => new Tuple <MaterializedOrder, MatchEdge>(MaterializedOrder.Pre, e))
.Union(
candidateTree.PostMatOutgoingEdges.Select(
e => new Tuple <MaterializedOrder, MatchEdge>(MaterializedOrder.Post, e)))
.ToList();
if (inEdges.Any())
{
unmaterializedNodeMapping.Remove(root);
foreach (var t in inEdges)
{
var order = t.Item1;
var edge = t.Item2;
edgeMaterializedDict[edge] = true;
}
}
if (outEdges.Any())
{
foreach (var t in outEdges)
{
var order = t.Item1;
var edge = t.Item2;
edgeMaterializedDict[edge] = true;
}
}
var unmatEdges = candidateTree.UnmaterializedEdges;
foreach (var unmatEdge in unmatEdges)
{
edgeMaterializedDict[unmatEdge] = false;;
var unmatNodeInEdges = unmaterializedNodeMapping.GetOrCreate(unmatEdge.SinkNode);
unmatNodeInEdges.Add(unmatEdge);
}
}
/// <summary>
/// Calculate the number used for adjusting the SQL Server estimation in the downsize function.
/// </summary>
/// <param name="component"></param>
/// <param name="tablfRef"></param>
/// <param name="joinTable"></param>
/// <param name="size"></param>
/// <param name="estimatedSize"></param>
/// <param name="shrinkSize"></param>
/// <param name="joinTableTuple"></param>
private static void AdjustEstimation(
MatchComponent component,
WTableReference tablfRef,
WQualifiedJoin joinTable,
double size,
double estimatedSize,
double shrinkSize,
Tuple<WQualifiedJoin,String> joinTableTuple)
{
const int sizeFactor = 10;
int estimateFactor = 0;
if (size > sizeFactor*estimatedSize)
{
estimateFactor = (int)Math.Ceiling(size / estimatedSize);
}
else if (sizeFactor*size < estimatedSize)
{
estimatedSize /= shrinkSize;
component.EstimateSize /= shrinkSize;
component.FatherListofDownSizeTable.Add(joinTableTuple);
estimateFactor = (int) Math.Ceiling(size/estimatedSize);
}
if (estimateFactor > 1)
{
WTableReference crossApplyTable = tablfRef;
int pow = (int) (Math.Floor(Math.Log(estimateFactor, 1000)) + 1);
int adjustValue = (int) Math.Pow(estimateFactor, 1.0/pow);
while (pow > 0)
{
crossApplyTable = new WUnqualifiedJoin
{
FirstTableRef = crossApplyTable,
SecondTableRef = new WSchemaObjectFunctionTableReference
{
SchemaObject = new WSchemaObjectName(
new Identifier {Value = "dbo"},
new Identifier {Value = "UpSizeFunction"}),
Parameters = new List<WScalarExpression>
{
new WValueExpression {Value = adjustValue.ToString()}
},
Alias = new Identifier
{
Value = Path.GetRandomFileName().Replace(".", "").Substring(0, 8),
}
},
UnqualifiedJoinType = UnqualifiedJoinType.CrossApply
};
pow--;
component.EstimateSize *= adjustValue;
}
joinTable.FirstTableRef = crossApplyTable;
}
}
/// <summary>
/// Calculate join costs and update components using optimal join method & order
/// </summary>
/// <param name="nodeUnitCandidate"></param>
/// <param name="component"></param>
/// <param name="nodeTable"></param>
/// <param name="componentTable"></param>
/// <param name="joinCondition"></param>
/// <param name="nodeDegrees"></param>
/// <param name="estimatedNodeUnitSize"></param>
/// <param name="estimatedSelectivity"></param>
/// <returns></returns>
private static WTableReference GetPlanAndUpdateCost(
OneHeightTree nodeUnitCandidate,
MatchComponent component,
WTableReference nodeTable,
WTableReference componentTable,
WBooleanExpression joinCondition,
double nodeDegrees,
double joinSelectivity,
double estimatedNodeUnitSize,
double estimatedSelectivity)
{
var nodeUnitSize = nodeUnitCandidate.TreeRoot.EstimatedRows * nodeDegrees;
var componentSize = component.Size;
var estimatedCompSize = component.EstimateSize;
var cost = nodeUnitSize + componentSize;
//var joinSelectivity =
// nodeWithJoinMapping.SelectivityProduct;
//nodeWithJoinMapping.ExponentialSelevtivityProduct;
WQualifiedJoin joinTable = new WQualifiedJoin
{
FirstTableRef = componentTable,
SecondTableRef = nodeTable,
JoinCondition = joinCondition,
QualifiedJoinType = QualifiedJoinType.Inner,
JoinHint = JoinHint.Hash
};
var node = nodeUnitCandidate.TreeRoot;
// If the node is already in the component, then only multiply the degree to get the size
double nodeUnitActualSize;
if (component.MaterializedNodeSplitCount[node] > 0)
{
nodeUnitActualSize = nodeDegrees;
var cEstEdge = Math.Pow(1000, component.EdgeMaterilizedDict.Count(e=>!e.Value));
var cSize = component.EstimateSize/cEstEdge;
var nSize = node.EstimatedRows;
if (nSize > cSize)
{
component.EstimateSize = estimatedNodeUnitSize*cEstEdge*estimatedSelectivity;
}
else
{
component.EstimateSize = component.EstimateSize*Math.Pow(1000, nodeUnitCandidate.UnmaterializedEdges.Count)*
estimatedSelectivity;
}
}
else
{
nodeUnitActualSize = nodeUnitSize;
component.EstimateSize *= estimatedNodeUnitSize * estimatedSelectivity;
}
//Update Size
component.Size *= nodeUnitActualSize * joinSelectivity;
// Update Cost
component.Cost += cost;
// Debug
#if DEBUG
//Trace.Write(component.NodeUnits.Count+" ");
//foreach (var n in component.NodeUnits.Where(e => e.Key != node.Node.ExposedName))
//{
// Trace.Write(n.Value.NodeRefName);
//}
//Trace.Write(component.NodeUnits[node.Node.ExposedName].NodeRefName+" ");
//Trace.Write(" "+(long)component.Cost+" "+(long)component.Size);
//Trace.Write(" ");
//foreach (var item in component.PopulatedEdgesName)
//{
// Trace.Write(item + " ");
//}
//Trace.Write("; ");
//foreach (var unpopulatedEdge in component.UnpopulatedEdges)
//{
// Trace.Write(unpopulatedEdge.Alias + " ");
//}
//Trace.WriteLine("");
#endif
// Update TableRef
// Only consider the size in the first join
if (component.MaterializedNodeSplitCount.Count == 2 && component.MaterializedNodeSplitCount.All(e => e.Value == 0))
{
var nodeInComp = component.MaterializedNodeSplitCount.Keys.First(e => e != node);
if (nodeUnitSize < componentSize)
{
joinTable.FirstTableRef = nodeTable;
joinTable.SecondTableRef = componentTable;
component.TotalMemory = component.DeltaMemory = nodeUnitSize;
component.EstimateTotalMemory = component.EstimateDeltaMemory = estimatedNodeUnitSize;
component.RightestTableRefSize = nodeInComp.EstimatedRows;
component.FatherOfRightestTableRef = new Tuple<WQualifiedJoin, String>(joinTable,
component.GetNodeRefName(nodeInComp));
AdjustEstimation(component, nodeTable, joinTable, nodeUnitSize, estimatedNodeUnitSize,
nodeUnitCandidate.TreeRoot.EstimatedRows, new Tuple<WQualifiedJoin, string>(joinTable, component.GetNodeRefName(node)));
}
else
{
component.TotalMemory = component.DeltaMemory = componentSize;
component.EstimateTotalMemory = component.EstimateDeltaMemory = component.EstimateSize;
component.RightestTableRefSize = nodeUnitCandidate.TreeRoot.EstimatedRows;
component.FatherOfRightestTableRef = new Tuple<WQualifiedJoin, String>(joinTable, component.GetNodeRefName(node));
AdjustEstimation(component, componentTable, joinTable, componentSize, estimatedCompSize,
nodeInComp.EstimatedRows, new Tuple<WQualifiedJoin, string>(joinTable, component.GetNodeRefName(nodeInComp)));
}
}
else
{
double sizeFactor = 5;//1000;
double maxMemory = 1e8;
double loopJoinInnerThreshold = 10000;
double loopJoinOuterThreshold = 1000000;
// Left Deep
if (componentSize*sizeFactor < nodeUnitSize)
{
var curDeltaMemory = componentSize;
component.TotalMemory = component.DeltaMemory + curDeltaMemory;
component.DeltaMemory = curDeltaMemory;
var curDeltaEstimateMemory = component.EstimateSize;
component.EstimateTotalMemory = component.EstimateDeltaMemory + curDeltaEstimateMemory;
component.EstimateDeltaMemory = curDeltaEstimateMemory;
// Adjust estimation in sql server
AdjustEstimation(component, componentTable, joinTable, componentSize, estimatedCompSize,
component.RightestTableRefSize, component.FatherOfRightestTableRef);
component.FatherOfRightestTableRef = new Tuple<WQualifiedJoin, string>(joinTable, component.GetNodeRefName(node));
component.RightestTableRefSize = nodeUnitCandidate.TreeRoot.EstimatedRows;
}
else
{
// Loop Join
if (
//((nodeUnitSize < loopJoinInnerThreshold /*&& componentSize < loopJoinOuterThreshold*/) || component.DeltaMemory + componentSize > maxMemory / 100) &&
((nodeUnitSize < loopJoinInnerThreshold && componentSize < loopJoinOuterThreshold) ||
component.DeltaMemory + componentSize > maxMemory) &&
nodeUnitCandidate.MaterializedEdges.Count==0)
{
component.TotalMemory = component.DeltaMemory;
component.EstimateTotalMemory = component.EstimateDeltaMemory;
joinTable.JoinHint = JoinHint.Loop;
component.EstimateSize = estimatedCompSize*estimatedNodeUnitSize/
nodeUnitCandidate.TreeRoot.TableRowCount;
}
// Right Deep
else
{
joinTable.FirstTableRef = nodeTable;
joinTable.SecondTableRef = componentTable;
AdjustEstimation(component, nodeTable, joinTable, nodeUnitSize, estimatedNodeUnitSize,
node.EstimatedRows, new Tuple<WQualifiedJoin, string>(joinTable, component.GetNodeRefName(node)));
component.TotalMemory += nodeUnitSize;
component.DeltaMemory = component.TotalMemory;
component.EstimateTotalMemory += estimatedNodeUnitSize;
component.EstimateDeltaMemory = component.EstimateTotalMemory;
}
}
}
return new WParenthesisTableReference
{
Table = joinTable
};
}
/// <summary>
/// Transit from current component to the new component in the next state given the Node Unit
/// </summary>
/// <param name="candidateTree"></param>
/// <param name="densityDict"></param>
/// <param name="subGraph"></param>
/// <param name="statisticsCalculator"></param>
/// <returns></returns>
public MatchComponent GetNextState(
OneHeightTree candidateTree,
Dictionary<string, double> densityDict,
IMatchJoinStatisticsCalculator statisticsCalculator)
{
var newComponent = new MatchComponent(this);
var root = candidateTree.TreeRoot;
WBooleanExpression joinCondition = null;
string nodeName = "";
// Update Nodes
if (newComponent.MaterializedNodeSplitCount.ContainsKey(root))
{
newComponent.MaterializedNodeSplitCount[root]++;
nodeName = newComponent.GetNodeRefName(root);
joinCondition = new WBooleanComparisonExpression
{
FirstExpr = new WColumnReferenceExpression
{
ColumnType = ColumnType.Regular,
MultiPartIdentifier = new WMultiPartIdentifier(
new Identifier {Value = root.RefAlias},
new Identifier {Value = "GlobalNodeId"}
),
},
SecondExpr = new WColumnReferenceExpression
{
ColumnType = ColumnType.Regular,
MultiPartIdentifier = new WMultiPartIdentifier(
new Identifier {Value = nodeName},
new Identifier {Value = "GlobalNodeId"}
),
},
ComparisonType = BooleanComparisonType.Equals
};
}
else
{
nodeName = root.RefAlias;
newComponent.Nodes.Add(root);
newComponent.MaterializedNodeSplitCount[root] = 0;
newComponent.StatisticsDict[root] = new ColumnStatistics {Selectivity = 1.0/root.TableRowCount};
}
// Constructs table reference
WTableReference nodeTable = new WNamedTableReference
{
Alias = new Identifier { Value = nodeName },
TableObjectName = root.TableObjectName
};
WTableReference compTable = newComponent.TableRef;
// Updates join conditions
double selectivity = 1.0;
double degrees = 1.0;
var DensityCount = new Dictionary<string, int>(StringComparer.CurrentCultureIgnoreCase);
List<MatchEdge> inEdges;
if (newComponent.UnmaterializedNodeMapping.TryGetValue(root, out inEdges))
{
var firstEdge = inEdges.First();
bool materialized = newComponent.EdgeMaterilizedDict[firstEdge];
newComponent.UnmaterializedNodeMapping.Remove(root);
selectivity *= 1.0/root.TableRowCount;
// Component materialized edge to root
if (materialized)
{
joinCondition = WBooleanBinaryExpression.Conjunction(joinCondition, new WBooleanComparisonExpression
{
FirstExpr = new WColumnReferenceExpression
{
ColumnType = ColumnType.Regular,
MultiPartIdentifier = new WMultiPartIdentifier(
new Identifier {Value = firstEdge.EdgeAlias},
new Identifier {Value = "Sink"}
),
},
SecondExpr = new WColumnReferenceExpression
{
ColumnType = ColumnType.Regular,
MultiPartIdentifier = new WMultiPartIdentifier(
new Identifier {Value = nodeName},
new Identifier {Value = "GlobalNodeId"}
)
},
ComparisonType = BooleanComparisonType.Equals
});
//var statistics = ColumnStatistics.UpdateHistogram(newComponent.StatisticsDict[root],
// new ColumnStatistics {Selectivity = 1.0/root.TableRowCount});
//selectivity *= statistics.Selectivity;
//newComponent.StatisticsDict[root] = statistics;
if (DensityCount.ContainsKey(root.TableObjectName.ToString()))
DensityCount[root.TableObjectName.ToString()]++;
else
DensityCount[root.TableObjectName.ToString()] = 1;
}
// Component unmaterialized edge to root
else
{
ColumnStatistics statistics = null;
foreach (var edge in inEdges)
{
// Update component table
compTable = SpanTableRef(compTable, edge, newComponent.GetNodeRefName(edge.SourceNode));
newComponent.EdgeMaterilizedDict[edge] = true;
joinCondition = WBooleanBinaryExpression.Conjunction(joinCondition,
new WBooleanComparisonExpression
{
FirstExpr = new WColumnReferenceExpression
{
ColumnType = ColumnType.Regular,
MultiPartIdentifier = new WMultiPartIdentifier(
new Identifier {Value = edge.EdgeAlias},
new Identifier {Value = "Sink"}
),
},
SecondExpr = new WColumnReferenceExpression
{
ColumnType = ColumnType.Regular,
MultiPartIdentifier = new WMultiPartIdentifier(
new Identifier {Value = nodeName},
new Identifier {Value = "GlobalNodeId"}
)
},
ComparisonType = BooleanComparisonType.Equals
});
statistics = ColumnStatistics.UpdateHistogram(statistics,
newComponent.Context.GetEdgeStatistics(edge));
selectivity *= statistics.Selectivity;
}
newComponent.StatisticsDict[root] = statistics;
if (DensityCount.ContainsKey(root.TableObjectName.ToString()))
DensityCount[root.TableObjectName.ToString()]+=inEdges.Count;
else
DensityCount[root.TableObjectName.ToString()] = inEdges.Count;
}
}
var jointEdges = candidateTree.MaterializedEdges;
int sinkToSinkCount = 0;
foreach (var jointEdge in jointEdges)
{
// Update node table
nodeTable = SpanTableRef(nodeTable, jointEdge, nodeName);
degrees *= jointEdge.AverageDegree;
newComponent.EdgeMaterilizedDict[jointEdge] = true;
var sinkNode = jointEdge.SinkNode;
// Leaf to component materialized node
if (newComponent.MaterializedNodeSplitCount.ContainsKey(sinkNode))
{
joinCondition = WBooleanBinaryExpression.Conjunction(joinCondition,
new WBooleanComparisonExpression
{
FirstExpr = new WColumnReferenceExpression
{
ColumnType = ColumnType.Regular,
MultiPartIdentifier = new WMultiPartIdentifier(
new Identifier {Value = jointEdge.EdgeAlias},
new Identifier {Value = "Sink"}
),
},
SecondExpr = new WColumnReferenceExpression
{
ColumnType = ColumnType.Regular,
MultiPartIdentifier = new WMultiPartIdentifier(
new Identifier {Value = sinkNode.RefAlias},
new Identifier {Value = "GlobalNodeId"}
)
},
ComparisonType = BooleanComparisonType.Equals
});
var statistics = ColumnStatistics.UpdateHistogram(newComponent.StatisticsDict[sinkNode],
newComponent.Context.GetEdgeStatistics(jointEdge));
selectivity *= statistics.Selectivity;
newComponent.StatisticsDict[sinkNode] = statistics;
if (DensityCount.ContainsKey(sinkNode.TableObjectName.ToString()))
DensityCount[sinkNode.TableObjectName.ToString()]++;
else
DensityCount[sinkNode.TableObjectName.ToString()] = 1;
}
// Leaf to component unmaterialized node
else
{
inEdges = newComponent.UnmaterializedNodeMapping[sinkNode];
var firstEdge = inEdges.First();
bool materlizedEdge = newComponent.EdgeMaterilizedDict[firstEdge];
// Leaf to materialized leaf
if (materlizedEdge)
{
joinCondition = WBooleanBinaryExpression.Conjunction(joinCondition,
new WBooleanComparisonExpression
{
FirstExpr = new WColumnReferenceExpression
{
ColumnType = ColumnType.Regular,
MultiPartIdentifier = new WMultiPartIdentifier(
new Identifier {Value = jointEdge.EdgeAlias},
new Identifier {Value = "Sink"}
),
},
SecondExpr = new WColumnReferenceExpression
{
ColumnType = ColumnType.Regular,
MultiPartIdentifier = new WMultiPartIdentifier(
new Identifier { Value = firstEdge.EdgeAlias},
new Identifier {Value = "Sink"}
)
},
ComparisonType = BooleanComparisonType.Equals
});
sinkToSinkCount++;
var statistics = ColumnStatistics.UpdateHistogram(newComponent.StatisticsDict[sinkNode],
newComponent.Context.GetEdgeStatistics(jointEdge));
selectivity *= statistics.Selectivity;
newComponent.StatisticsDict[sinkNode] = statistics;
}
// Leaf to unmaterialized leaf
else
{
ColumnStatistics compSinkNodeStatistics = null;
foreach (var inEdge in inEdges)
{
compTable = SpanTableRef(compTable, inEdge, newComponent.GetNodeRefName(inEdge.SourceNode));
newComponent.EdgeMaterilizedDict[inEdge] = true;
joinCondition = WBooleanBinaryExpression.Conjunction(joinCondition,
new WBooleanComparisonExpression
{
FirstExpr = new WColumnReferenceExpression
{
ColumnType = ColumnType.Regular,
MultiPartIdentifier = new WMultiPartIdentifier(
new Identifier { Value = jointEdge.EdgeAlias },
new Identifier { Value = "Sink" }
),
},
SecondExpr = new WColumnReferenceExpression
{
ColumnType = ColumnType.Regular,
MultiPartIdentifier = new WMultiPartIdentifier(
new Identifier { Value = inEdge.EdgeAlias },
new Identifier { Value = "Sink" }
)
},
ComparisonType = BooleanComparisonType.Equals
});
sinkToSinkCount++;
var leafToLeafStatistics = statisticsCalculator.GetLeafToLeafStatistics(jointEdge, inEdge);
selectivity *= leafToLeafStatistics.Selectivity;
compSinkNodeStatistics =
ColumnStatistics.UpdateHistogram(compSinkNodeStatistics,
newComponent.Context.GetEdgeStatistics(inEdge));
}
newComponent.StatisticsDict[sinkNode] = compSinkNodeStatistics;
}
}
}
var unmatEdges = candidateTree.UnmaterializedEdges;
foreach (var unmatEdge in unmatEdges)
{
newComponent.EdgeMaterilizedDict[unmatEdge] = false;
newComponent.Nodes.Add(unmatEdge.SinkNode);
var sinkNodeInEdges = newComponent.UnmaterializedNodeMapping.GetOrCreate(unmatEdge.SinkNode);
sinkNodeInEdges.Add(unmatEdge);
degrees *= unmatEdge.AverageDegree;
}
// Calculate Estimated Join Selectivity & Estimated Node Size
double estimatedSelectity = 1.0;
int count = 0;
bool sinkJoin = false;
foreach (var item in densityDict.Where(e => DensityCount.ContainsKey(e.Key)))
{
var density = item.Value;
var curJoinCount = DensityCount[item.Key];
var curJoinSelectitivy = Math.Pow(density, 2 - Math.Pow(2, 1 - curJoinCount));
if (!sinkJoin && ColumnStatistics.DefaultDensity < density)
{
var curSinkJoinSelectivity = Math.Pow(ColumnStatistics.DefaultDensity,
2 - Math.Pow(2, 1 - sinkToSinkCount));
estimatedSelectity *= Math.Pow(curSinkJoinSelectivity, Math.Pow(2, -count));
count += sinkToSinkCount;
sinkJoin = true;
}
estimatedSelectity *= Math.Pow(curJoinSelectitivy, Math.Pow(2, -count));
count += curJoinCount;
}
var estimatedNodeUnitSize = root.EstimatedRows*
Math.Pow(1000, candidateTree.MaterializedEdges.Count + candidateTree.UnmaterializedEdges.Count);
// Update Table Reference
newComponent.TableRef = GetPlanAndUpdateCost(candidateTree, newComponent, nodeTable, compTable, joinCondition,
degrees, selectivity, estimatedNodeUnitSize, estimatedSelectity);
return newComponent;
}
public IEnumerable <CandidateJoinUnit> GetCandidateUnits(IEnumerable <OneHeightTree> trees,
MatchComponent component, Dictionary <string, MatchEdge> revEdgeDict)
{
foreach (var tree in trees)
{
//bool singleNode = treeTuple.Item2;
var root = tree.TreeRoot;
List <MatchEdge> inEdges;
component.UnmaterializedNodeMapping.TryGetValue(root, out inEdges);
var outEdges = new List <MatchEdge>();
var unpopEdges = new List <MatchEdge>();
foreach (var edge in tree.Edges)
{
if (component.Nodes.Contains(edge.SinkNode))
{
outEdges.Add(edge);
}
else
{
unpopEdges.Add(edge);
}
}
var rawEdges = new Dictionary <string, Tuple <MatchEdge, EdgeDir> >();
var extInEdges = new Dictionary <string, MatchEdge>();
if (inEdges != null)
{
rawEdges = inEdges.ToDictionary(edge => edge.EdgeAlias,
edge => new Tuple <MatchEdge, EdgeDir>(edge, EdgeDir.In));
extInEdges = inEdges.ToDictionary(edge => edge.EdgeAlias);
}
foreach (var edge in outEdges)
{
var key = edge.EdgeAlias;
rawEdges.Add(key, new Tuple <MatchEdge, EdgeDir>(edge, EdgeDir.Out));
if (edge.HasReversedEdge)
{
extInEdges.Add(key, revEdgeDict[key]);
}
}
var extOutEdges = outEdges.ToDictionary(edge => edge.EdgeAlias);
if (inEdges != null)
{
foreach (var key in inEdges.Where(edge => edge.HasReversedEdge).Select(edge => edge.EdgeAlias))
{
extOutEdges.Add(key, revEdgeDict[key]);
}
}
var sortedExtInEdges = (from entry in extInEdges
orderby entry.Value.AverageDegree ascending
select entry).ToList();
var sortedExtOutEdges = (from entry in extOutEdges
orderby entry.Value.AverageDegree ascending
select entry).ToList();
// plan type 1: A => B, Loop Join, might use Hash if |A|/|B| is too large
if (sortedExtInEdges.Any())
{
var preMatInEdges = new Dictionary <string, MatchEdge>
{
{ sortedExtInEdges[0].Key, sortedExtInEdges[0].Value }
};
for (var i = 1; i < sortedExtInEdges.Count; ++i)
{
if (sortedExtInEdges[i].Value.AverageDegree < 1)
{
preMatInEdges.Add(sortedExtInEdges[i].Key, sortedExtInEdges[i].Value);
}
}
var postMatEdges =
rawEdges.Where(entry => !preMatInEdges.ContainsKey(entry.Key))
.Select(entry => entry.Value).ToList();
for (var i = 0; i < postMatEdges.Count; ++i)
{
var t = postMatEdges[i];
var edge = t.Item1;
var dir = t.Item2;
if (!edge.HasReversedEdge)
{
continue;
}
var revEdge = revEdgeDict[edge.EdgeAlias];
if (revEdge.AverageDegree < edge.AverageDegree)
{
postMatEdges[i] = new Tuple <MatchEdge, EdgeDir>(revEdge,
dir == EdgeDir.In ? EdgeDir.Out : EdgeDir.In);
}
}
yield return(new CandidateJoinUnit
{
TreeRoot = root,
PreMatIncomingEdges = preMatInEdges.Select(entry => entry.Value)
.OrderBy(edge => edge.AverageDegree).ToList(),
PreMatOutgoingEdges = new List <MatchEdge>(),
PostMatIncomingEdges = postMatEdges.Where(entry => entry.Item2 == EdgeDir.In)
.Select(entry => entry.Item1).ToList(),
PostMatOutgoingEdges = postMatEdges.Where(entry => entry.Item2 == EdgeDir.Out)
.Select(entry => entry.Item1).ToList(),
UnmaterializedEdges = unpopEdges,
JoinHint = JoinHint.Loop,
});
}
// plan type 2: A <= B, Hash join
else if (sortedExtOutEdges.Any())
{
var preMatOutEdges = new Dictionary <string, MatchEdge>
{
{ sortedExtOutEdges[0].Key, sortedExtOutEdges[0].Value }
};
for (var i = 1; i < sortedExtOutEdges.Count; ++i)
{
if (sortedExtOutEdges[i].Value.AverageDegree < 1)
{
preMatOutEdges.Add(sortedExtOutEdges[i].Key, sortedExtOutEdges[i].Value);
}
}
var postMatEdges =
rawEdges.Where(entry => !preMatOutEdges.ContainsKey(entry.Key))
.Select(entry => entry.Value).ToList();
for (var i = 0; i < postMatEdges.Count; ++i)
{
var t = postMatEdges[i];
var edge = t.Item1;
var dir = t.Item2;
if (!edge.HasReversedEdge)
{
continue;
}
var revEdge = revEdgeDict[edge.EdgeAlias];
if (revEdge.AverageDegree < edge.AverageDegree)
{
postMatEdges[i] = new Tuple <MatchEdge, EdgeDir>(revEdge,
dir == EdgeDir.In ? EdgeDir.Out : EdgeDir.In);
}
}
yield return(new CandidateJoinUnit
{
TreeRoot = root,
PreMatIncomingEdges = new List <MatchEdge>(),
PreMatOutgoingEdges = preMatOutEdges.Select(entry => entry.Value).ToList(),
PostMatIncomingEdges = postMatEdges.Where(entry => entry.Item2 == EdgeDir.In)
.Select(entry => entry.Item1).ToList(),
PostMatOutgoingEdges = postMatEdges.Where(entry => entry.Item2 == EdgeDir.Out)
.Select(entry => entry.Item1).ToList(),
UnmaterializedEdges = unpopEdges,
JoinHint = JoinHint.Hash,
});
}
}
}
/// <summary>
/// Calculate join costs and update components using optimal join method & order
/// </summary>
/// <param name="nodeUnitCandidate"></param>
/// <param name="component"></param>
/// <param name="nodeTable"></param>
/// <param name="componentTable"></param>
/// <param name="joinCondition"></param>
/// <param name="nodeDegrees"></param>
/// <param name="estimatedNodeUnitSize"></param>
/// <param name="estimatedSelectivity"></param>
/// <returns></returns>
private static WTableReference GetPlanAndUpdateCost(
OneHeightTree nodeUnitCandidate,
MatchComponent component,
WTableReference nodeTable,
WTableReference componentTable,
WBooleanExpression joinCondition,
double nodeDegrees,
double joinSelectivity,
double estimatedNodeUnitSize,
double estimatedSelectivity)
{
var nodeUnitSize = nodeUnitCandidate.TreeRoot.EstimatedRows * nodeDegrees;
var componentSize = component.Size;
var estimatedCompSize = component.EstimateSize;
var cost = nodeUnitSize + componentSize;
//var joinSelectivity =
// nodeWithJoinMapping.SelectivityProduct;
//nodeWithJoinMapping.ExponentialSelevtivityProduct;
WQualifiedJoin joinTable = new WQualifiedJoin
{
FirstTableRef = componentTable,
SecondTableRef = nodeTable,
JoinCondition = joinCondition,
QualifiedJoinType = QualifiedJoinType.Inner,
JoinHint = JoinHint.Hash
};
var node = nodeUnitCandidate.TreeRoot;
// If the node is already in the component, then only multiply the degree to get the size
double nodeUnitActualSize;
if (component.MaterializedNodeSplitCount[node] > 0)
{
nodeUnitActualSize = nodeDegrees;
var cEstEdge = Math.Pow(1000, component.EdgeMaterilizedDict.Count(e => !e.Value));
var cSize = component.EstimateSize / cEstEdge;
var nSize = node.EstimatedRows;
if (nSize > cSize)
{
component.EstimateSize = estimatedNodeUnitSize * cEstEdge * estimatedSelectivity;
}
else
{
component.EstimateSize = component.EstimateSize * Math.Pow(1000, nodeUnitCandidate.UnmaterializedEdges.Count) *
estimatedSelectivity;
}
}
else
{
nodeUnitActualSize = nodeUnitSize;
component.EstimateSize *= estimatedNodeUnitSize * estimatedSelectivity;
}
//Update Size
component.Size *= nodeUnitActualSize * joinSelectivity;
// Update Cost
component.Cost += cost;
// Debug
#if DEBUG
//Trace.Write(component.NodeUnits.Count+" ");
//foreach (var n in component.NodeUnits.Where(e => e.Key != node.Node.ExposedName))
//{
// Trace.Write(n.Value.NodeRefName);
//}
//Trace.Write(component.NodeUnits[node.Node.ExposedName].NodeRefName+" ");
//Trace.Write(" "+(long)component.Cost+" "+(long)component.Size);
//Trace.Write(" ");
//foreach (var item in component.PopulatedEdgesName)
//{
// Trace.Write(item + " ");
//}
//Trace.Write("; ");
//foreach (var unpopulatedEdge in component.UnpopulatedEdges)
//{
// Trace.Write(unpopulatedEdge.Alias + " ");
//}
//Trace.WriteLine("");
#endif
// Update TableRef
// Only consider the size in the first join
if (component.MaterializedNodeSplitCount.Count == 2 && component.MaterializedNodeSplitCount.All(e => e.Value == 0))
{
var nodeInComp = component.MaterializedNodeSplitCount.Keys.First(e => e != node);
if (nodeUnitSize < componentSize)
{
joinTable.FirstTableRef = nodeTable;
joinTable.SecondTableRef = componentTable;
component.TotalMemory = component.DeltaMemory = nodeUnitSize;
component.EstimateTotalMemory = component.EstimateDeltaMemory = estimatedNodeUnitSize;
component.RightestTableRefSize = nodeInComp.EstimatedRows;
component.FatherOfRightestTableRef = new Tuple <WQualifiedJoin, String>(joinTable,
component.GetNodeRefName(nodeInComp));
AdjustEstimation(component, nodeTable, joinTable, nodeUnitSize, estimatedNodeUnitSize,
nodeUnitCandidate.TreeRoot.EstimatedRows, new Tuple <WQualifiedJoin, string>(joinTable, component.GetNodeRefName(node)));
}
else
{
component.TotalMemory = component.DeltaMemory = componentSize;
component.EstimateTotalMemory = component.EstimateDeltaMemory = component.EstimateSize;
component.RightestTableRefSize = nodeUnitCandidate.TreeRoot.EstimatedRows;
component.FatherOfRightestTableRef = new Tuple <WQualifiedJoin, String>(joinTable, component.GetNodeRefName(node));
AdjustEstimation(component, componentTable, joinTable, componentSize, estimatedCompSize,
nodeInComp.EstimatedRows, new Tuple <WQualifiedJoin, string>(joinTable, component.GetNodeRefName(nodeInComp)));
}
}
else
{
double sizeFactor = 5;//1000;
double maxMemory = 1e8;
double loopJoinInnerThreshold = 10000;
double loopJoinOuterThreshold = 1000000;
// Left Deep
if (componentSize * sizeFactor < nodeUnitSize)
{
var curDeltaMemory = componentSize;
component.TotalMemory = component.DeltaMemory + curDeltaMemory;
component.DeltaMemory = curDeltaMemory;
var curDeltaEstimateMemory = component.EstimateSize;
component.EstimateTotalMemory = component.EstimateDeltaMemory + curDeltaEstimateMemory;
component.EstimateDeltaMemory = curDeltaEstimateMemory;
// Adjust estimation in sql server
AdjustEstimation(component, componentTable, joinTable, componentSize, estimatedCompSize,
component.RightestTableRefSize, component.FatherOfRightestTableRef);
component.FatherOfRightestTableRef = new Tuple <WQualifiedJoin, string>(joinTable, component.GetNodeRefName(node));
component.RightestTableRefSize = nodeUnitCandidate.TreeRoot.EstimatedRows;
}
else
{
// Loop Join
if (
//((nodeUnitSize < loopJoinInnerThreshold /*&& componentSize < loopJoinOuterThreshold*/) || component.DeltaMemory + componentSize > maxMemory / 100) &&
((nodeUnitSize < loopJoinInnerThreshold && componentSize < loopJoinOuterThreshold) ||
component.DeltaMemory + componentSize > maxMemory) &&
nodeUnitCandidate.MaterializedEdges.Count == 0)
{
component.TotalMemory = component.DeltaMemory;
component.EstimateTotalMemory = component.EstimateDeltaMemory;
joinTable.JoinHint = JoinHint.Loop;
component.EstimateSize = estimatedCompSize * estimatedNodeUnitSize /
nodeUnitCandidate.TreeRoot.TableRowCount;
}
// Right Deep
else
{
joinTable.FirstTableRef = nodeTable;
joinTable.SecondTableRef = componentTable;
AdjustEstimation(component, nodeTable, joinTable, nodeUnitSize, estimatedNodeUnitSize,
node.EstimatedRows, new Tuple <WQualifiedJoin, string>(joinTable, component.GetNodeRefName(node)));
component.TotalMemory += nodeUnitSize;
component.DeltaMemory = component.TotalMemory;
component.EstimateTotalMemory += estimatedNodeUnitSize;
component.EstimateDeltaMemory = component.EstimateTotalMemory;
}
}
}
return(new WParenthesisTableReference
{
Table = joinTable
});
}
/// <summary>
/// Transit from current component to the new component in the next state given the Node Unit
/// </summary>
/// <param name="candidateTree"></param>
/// <param name="statisticsCalculator"></param>
/// <param name="metaData"></param>
/// <param name="srcNodeStatisticsDict"></param>
/// <returns></returns>
public MatchComponent GetNextState(
CandidateJoinUnit candidateTree,
IMatchJoinStatisticsCalculator statisticsCalculator,
GraphMetaData metaData,
Dictionary<Tuple<string, bool>, Statistics> srcNodeStatisticsDict)
{
// Deep copy the component
var newComponent = new MatchComponent(this);
// Constrcuts join conditions and retrieves join selectivity
double preJoinSelectivity, postJoinSelectivity, sqlEstimatedJoinSelectivity;
var joinCondition = newComponent.ConstructJoinCondition(candidateTree, statisticsCalculator, metaData, srcNodeStatisticsDict,
out preJoinSelectivity, out postJoinSelectivity, out sqlEstimatedJoinSelectivity);
// Constructs physical join method and join table references
newComponent.ConstructPhysicalJoinAndUpdateCost(candidateTree, joinCondition,
preJoinSelectivity, postJoinSelectivity, sqlEstimatedJoinSelectivity, metaData);
return newComponent;
}
/// <summary>
/// Get a full one height tree with joint edges and unmaterlized edges,
/// returns a tuple whose first item is the one height tree and the second item
/// indicates whether the one height tree only joins with the component's materialized
/// node on its root.
/// </summary>
/// <param name="graph"></param>
/// <param name="component"></param>
/// <returns></returns>
private IEnumerable<Tuple<OneHeightTree, bool>> GetNodeUnits(ConnectedComponent graph, MatchComponent component)
{
var nodes = graph.Nodes;
foreach (var node in nodes.Values.Where(e => !graph.IsTailNode[e]))
{
var remainingEdges = node.Neighbors.Where(e => !component.EdgeMaterilizedDict.ContainsKey(e)).ToList();
// If there exists a component's edge pointing to the 1-heright tree's root
// or a 1-height tree's edge pointing to the component's node, then generates a valid
// 1-height tree with edges. Otherwise, if a node can be joint to component as a single
// split node with unmaterialized edges, generates a 1-height tree with a tag set to true.
if (component.UnmaterializedNodeMapping.ContainsKey(node) ||
remainingEdges.Any(e => component.Nodes.Contains(e.SinkNode)))
{
yield return new Tuple<OneHeightTree, bool>(new OneHeightTree
{
TreeRoot = node,
Edges = remainingEdges
}, false);
}
else if (remainingEdges.Count > 0 && component.MaterializedNodeSplitCount.Count > 1 &&
component.MaterializedNodeSplitCount.ContainsKey(node))
{
yield return new Tuple<OneHeightTree, bool>(new OneHeightTree
{
TreeRoot = node,
Edges = remainingEdges
}, true);
}
}
}
/// <summary>
/// Get a full one height tree with joint edges and unmaterlized edges,
/// returns a tuple whose first item is the one height tree and the second item
/// indicates whether the one height tree only joins with the component's materialized
/// node on its root.
/// </summary>
/// <param name="graph"></param>
/// <param name="component"></param>
/// <returns></returns>
private IEnumerable<OneHeightTree> GetNodeUnits(ConnectedComponent graph, MatchComponent component)
{
//var nodes = graph.Nodes;
//foreach (var node in graph.Nodes.Values)
foreach (var node in graph.Nodes.Values.Where(n => !component.Nodes.Contains(n)))
{
var remainingEdges = node.Neighbors.Where(e => !component.EdgeMaterilizedDict.ContainsKey(e)).ToList();
if (component.UnmaterializedNodeMapping.ContainsKey(node) ||
remainingEdges.Any(e => component.Nodes.Contains(e.SinkNode)))
{
yield return new OneHeightTree
{
TreeRoot = node,
Edges = remainingEdges
};
}
}
}
private CandidateJoinUnit GetCandidateUnits(OneHeightTree tree, MatchComponent component)
{
var nodeMatEdgesDict = component.NodeToMaterializedEdgesDict;
var revEdgeDict = Graph.ReversedEdgeDict;
var root = tree.TreeRoot;
nodeMatEdgesDict[root.NodeAlias] = new List <Tuple <MatchEdge, MaterializedEdgeType> >();
List <MatchEdge> inEdges;
component.UnmaterializedNodeMapping.TryGetValue(root, out inEdges);
var outEdges = new List <MatchEdge>();
var unpopEdges = new List <MatchEdge>();
foreach (var edge in tree.Edges)
{
if (component.Nodes.Contains(edge.SinkNode))
{
outEdges.Add(edge);
}
else
{
unpopEdges.Add(edge);
}
}
var rawEdges = new Dictionary <string, Tuple <MatchEdge, EdgeDir> >();
var extInEdges = new Dictionary <string, MatchEdge>();
if (inEdges != null)
{
rawEdges = inEdges.ToDictionary(edge => edge.EdgeAlias,
edge => new Tuple <MatchEdge, EdgeDir>(edge, EdgeDir.In));
extInEdges = inEdges.ToDictionary(edge => edge.EdgeAlias);
}
foreach (var edge in outEdges)
{
var key = edge.EdgeAlias;
rawEdges.Add(key, new Tuple <MatchEdge, EdgeDir>(edge, EdgeDir.Out));
extInEdges.Add(key, revEdgeDict[key]);
}
if (extInEdges.Any())
{
var firstEdge = extInEdges.FirstOrDefault(e => e.Value.IsReversed == false);
if (firstEdge.Value == null)
{
firstEdge = extInEdges.First();
}
var preMatInEdges = new Dictionary <string, MatchEdge>
{
{ firstEdge.Key, firstEdge.Value }
};
var postMatEdges = rawEdges.Where(entry => !preMatInEdges.ContainsKey(entry.Key))
.Select(entry => entry.Value).ToList();
var postMatIncomingEdges = postMatEdges.Where(entry => entry.Item2 == EdgeDir.In)
.Select(entry => entry.Item1).ToList();
var postMatOutgoingEdges = postMatEdges.Where(entry => entry.Item2 == EdgeDir.Out)
.Select(entry => entry.Item1).ToList();
nodeMatEdgesDict[firstEdge.Value.SourceNode.NodeAlias].Add(
new Tuple <MatchEdge, MaterializedEdgeType>(firstEdge.Value, MaterializedEdgeType.TraversalEdge));
foreach (var t in postMatEdges)
{
var edge = t.Item1;
var type = t.Item2 == EdgeDir.In
? MaterializedEdgeType.RemainingEdge
: MaterializedEdgeType.ReverseCheckEdge;
nodeMatEdgesDict[edge.SourceNode.NodeAlias].Add(new Tuple <MatchEdge, MaterializedEdgeType>(edge, type));
}
return(new CandidateJoinUnit
{
TreeRoot = root,
PreMatIncomingEdges = preMatInEdges.Select(entry => entry.Value).ToList(),
PreMatOutgoingEdges = new List <MatchEdge>(),
PostMatIncomingEdges = postMatIncomingEdges,
PostMatOutgoingEdges = postMatOutgoingEdges,
UnmaterializedEdges = unpopEdges,
});
}
else
{
throw new GraphViewException("This graph pattern is not yet supported.");
}
}
/// <summary>
/// Calculate the number used for adjusting the SQL Server estimation in the downsize function.
/// </summary>
/// <param name="component"></param>
/// <param name="tablfRef"></param>
/// <param name="joinTable"></param>
/// <param name="size"></param>
/// <param name="estimatedSize"></param>
/// <param name="shrinkSize"></param>
/// <param name="joinTableTuple"></param>
private static void AdjustEstimation(
MatchComponent component,
WTableReference tablfRef,
WQualifiedJoin joinTable,
double size,
double estimatedSize,
double shrinkSize,
Tuple <WQualifiedJoin, String> joinTableTuple)
{
const int sizeFactor = 10;
int estimateFactor = 0;
if (size > sizeFactor * estimatedSize)
{
estimateFactor = (int)Math.Ceiling(size / estimatedSize);
}
else if (sizeFactor * size < estimatedSize)
{
estimatedSize /= shrinkSize;
component.EstimateSize /= shrinkSize;
component.FatherListofDownSizeTable.Add(joinTableTuple);
estimateFactor = (int)Math.Ceiling(size / estimatedSize);
}
if (estimateFactor > 1)
{
WTableReference crossApplyTable = tablfRef;
int pow = (int)(Math.Floor(Math.Log(estimateFactor, 1000)) + 1);
int adjustValue = (int)Math.Pow(estimateFactor, 1.0 / pow);
while (pow > 0)
{
crossApplyTable = new WUnqualifiedJoin
{
FirstTableRef = crossApplyTable,
SecondTableRef = new WSchemaObjectFunctionTableReference
{
SchemaObject = new WSchemaObjectName(
new Identifier {
Value = "dbo"
},
new Identifier {
Value = "UpSizeFunction"
}),
Parameters = new List <WScalarExpression>
{
new WValueExpression {
Value = adjustValue.ToString()
}
},
Alias = new Identifier
{
Value = Path.GetRandomFileName().Replace(".", "").Substring(0, 8),
}
},
UnqualifiedJoinType = UnqualifiedJoinType.CrossApply
};
pow--;
component.EstimateSize *= adjustValue;
}
joinTable.FirstTableRef = crossApplyTable;
}
}
private void UpdateComponent(MatchComponent curComponent, CandidateJoinUnit candidateTree)
{
Dictionary <string, MatchNode> nodes = curComponent.Nodes;
Dictionary <MatchEdge, bool> edgeMaterializedDict = curComponent.EdgeMaterilizedDict;
Dictionary <MatchNode, List <MatchEdge> > unmaterializedNodeMapping = curComponent.UnmaterializedNodeMapping;
MatchNode root = candidateTree.TreeRoot;
if (!nodes.ContainsKey(root.NodeAlias))
{
nodes.Add(root.NodeAlias, new MatchNode(root));
}
curComponent.MaterializedNodeSplitCount[root] = 0;
List <Tuple <MaterializedOrder, MatchEdge> > inEdges =
candidateTree.PreMatIncomingEdges.Select(
e => new Tuple <MaterializedOrder, MatchEdge>(MaterializedOrder.Pre, e))
.Union(
candidateTree.PostMatIncomingEdges.Select(
e => new Tuple <MaterializedOrder, MatchEdge>(MaterializedOrder.Post, e)))
.ToList();
List <Tuple <MaterializedOrder, MatchEdge> > outEdges =
candidateTree.PreMatOutgoingEdges.Select(
e => new Tuple <MaterializedOrder, MatchEdge>(MaterializedOrder.Pre, e))
.Union(
candidateTree.PostMatOutgoingEdges.Select(
e => new Tuple <MaterializedOrder, MatchEdge>(MaterializedOrder.Post, e)))
.ToList();
if (inEdges.Any())
{
unmaterializedNodeMapping.Remove(root);
foreach (Tuple <MaterializedOrder, MatchEdge> t in inEdges)
{
MaterializedOrder order = t.Item1;
MatchEdge edge = t.Item2;
edgeMaterializedDict[edge] = true;
List <string> adjListProperties = this.PopulateAdjacencyListProperties(edge);
MatchNode node = curComponent.Nodes[edge.SourceNode.NodeAlias];
foreach (string adjListProperty in adjListProperties)
{
node.Properties.Add(adjListProperty);
}
}
}
if (outEdges.Any())
{
foreach (Tuple <MaterializedOrder, MatchEdge> t in outEdges)
{
MaterializedOrder order = t.Item1;
MatchEdge edge = t.Item2;
edgeMaterializedDict[edge] = true;
List <string> adjListProperties = this.PopulateAdjacencyListProperties(edge);
MatchNode node = curComponent.Nodes[edge.SourceNode.NodeAlias];
foreach (string adjListProperty in adjListProperties)
{
node.Properties.Add(adjListProperty);
}
}
}
List <MatchEdge> unmatEdges = candidateTree.UnmaterializedEdges;
foreach (MatchEdge unmatEdge in unmatEdges)
{
edgeMaterializedDict[unmatEdge] = false;;
List <MatchEdge> unmatNodeInEdges = unmaterializedNodeMapping.GetOrCreate(unmatEdge.SinkNode);
unmatNodeInEdges.Add(unmatEdge);
}
}
/// <summary>
/// Transit from current component to the new component in the next state given the Node Unit
/// </summary>
/// <param name="candidateTree"></param>
/// <param name="densityDict"></param>
/// <param name="subGraph"></param>
/// <param name="statisticsCalculator"></param>
/// <returns></returns>
public MatchComponent GetNextState(
OneHeightTree candidateTree,
Dictionary <string, double> densityDict,
IMatchJoinStatisticsCalculator statisticsCalculator)
{
var newComponent = new MatchComponent(this);
var root = candidateTree.TreeRoot;
WBooleanExpression joinCondition = null;
string nodeName = "";
// Update Nodes
if (newComponent.MaterializedNodeSplitCount.ContainsKey(root))
{
newComponent.MaterializedNodeSplitCount[root]++;
nodeName = newComponent.GetNodeRefName(root);
joinCondition = new WBooleanComparisonExpression
{
FirstExpr = new WColumnReferenceExpression
{
ColumnType = ColumnType.Regular,
MultiPartIdentifier = new WMultiPartIdentifier(
new Identifier {
Value = root.RefAlias
},
new Identifier {
Value = "GlobalNodeId"
}
),
},
SecondExpr = new WColumnReferenceExpression
{
ColumnType = ColumnType.Regular,
MultiPartIdentifier = new WMultiPartIdentifier(
new Identifier {
Value = nodeName
},
new Identifier {
Value = "GlobalNodeId"
}
),
},
ComparisonType = BooleanComparisonType.Equals
};
}
else
{
nodeName = root.RefAlias;
newComponent.Nodes.Add(root);
newComponent.MaterializedNodeSplitCount[root] = 0;
newComponent.StatisticsDict[root] = new ColumnStatistics {
Selectivity = 1.0 / root.TableRowCount
};
}
// Constructs table reference
WTableReference nodeTable = new WNamedTableReference
{
Alias = new Identifier {
Value = nodeName
},
TableObjectName = root.TableObjectName
};
WTableReference compTable = newComponent.TableRef;
// Updates join conditions
double selectivity = 1.0;
double degrees = 1.0;
var DensityCount = new Dictionary <string, int>(StringComparer.CurrentCultureIgnoreCase);
List <MatchEdge> inEdges;
if (newComponent.UnmaterializedNodeMapping.TryGetValue(root, out inEdges))
{
var firstEdge = inEdges.First();
bool materialized = newComponent.EdgeMaterilizedDict[firstEdge];
newComponent.UnmaterializedNodeMapping.Remove(root);
selectivity *= 1.0 / root.TableRowCount;
// Component materialized edge to root
if (materialized)
{
joinCondition = WBooleanBinaryExpression.Conjunction(joinCondition, new WBooleanComparisonExpression
{
FirstExpr = new WColumnReferenceExpression
{
ColumnType = ColumnType.Regular,
MultiPartIdentifier = new WMultiPartIdentifier(
new Identifier {
Value = firstEdge.EdgeAlias
},
new Identifier {
Value = "Sink"
}
),
},
SecondExpr = new WColumnReferenceExpression
{
ColumnType = ColumnType.Regular,
MultiPartIdentifier = new WMultiPartIdentifier(
new Identifier {
Value = nodeName
},
new Identifier {
Value = "GlobalNodeId"
}
)
},
ComparisonType = BooleanComparisonType.Equals
});
//var statistics = ColumnStatistics.UpdateHistogram(newComponent.StatisticsDict[root],
// new ColumnStatistics {Selectivity = 1.0/root.TableRowCount});
//selectivity *= statistics.Selectivity;
//newComponent.StatisticsDict[root] = statistics;
if (DensityCount.ContainsKey(root.TableObjectName.ToString()))
{
DensityCount[root.TableObjectName.ToString()]++;
}
else
{
DensityCount[root.TableObjectName.ToString()] = 1;
}
}
// Component unmaterialized edge to root
else
{
ColumnStatistics statistics = null;
foreach (var edge in inEdges)
{
// Update component table
compTable = SpanTableRef(compTable, edge, newComponent.GetNodeRefName(edge.SourceNode));
newComponent.EdgeMaterilizedDict[edge] = true;
joinCondition = WBooleanBinaryExpression.Conjunction(joinCondition,
new WBooleanComparisonExpression
{
FirstExpr = new WColumnReferenceExpression
{
ColumnType = ColumnType.Regular,
MultiPartIdentifier = new WMultiPartIdentifier(
new Identifier {
Value = edge.EdgeAlias
},
new Identifier {
Value = "Sink"
}
),
},
SecondExpr = new WColumnReferenceExpression
{
ColumnType = ColumnType.Regular,
MultiPartIdentifier = new WMultiPartIdentifier(
new Identifier {
Value = nodeName
},
new Identifier {
Value = "GlobalNodeId"
}
)
},
ComparisonType = BooleanComparisonType.Equals
});
statistics = ColumnStatistics.UpdateHistogram(statistics,
newComponent.Context.GetEdgeStatistics(edge));
selectivity *= statistics.Selectivity;
}
newComponent.StatisticsDict[root] = statistics;
if (DensityCount.ContainsKey(root.TableObjectName.ToString()))
{
DensityCount[root.TableObjectName.ToString()] += inEdges.Count;
}
else
{
DensityCount[root.TableObjectName.ToString()] = inEdges.Count;
}
}
}
var jointEdges = candidateTree.MaterializedEdges;
int sinkToSinkCount = 0;
foreach (var jointEdge in jointEdges)
{
// Update node table
nodeTable = SpanTableRef(nodeTable, jointEdge, nodeName);
degrees *= jointEdge.AverageDegree;
newComponent.EdgeMaterilizedDict[jointEdge] = true;
var sinkNode = jointEdge.SinkNode;
// Leaf to component materialized node
if (newComponent.MaterializedNodeSplitCount.ContainsKey(sinkNode))
{
joinCondition = WBooleanBinaryExpression.Conjunction(joinCondition,
new WBooleanComparisonExpression
{
FirstExpr = new WColumnReferenceExpression
{
ColumnType = ColumnType.Regular,
MultiPartIdentifier = new WMultiPartIdentifier(
new Identifier {
Value = jointEdge.EdgeAlias
},
new Identifier {
Value = "Sink"
}
),
},
SecondExpr = new WColumnReferenceExpression
{
ColumnType = ColumnType.Regular,
MultiPartIdentifier = new WMultiPartIdentifier(
new Identifier {
Value = sinkNode.RefAlias
},
new Identifier {
Value = "GlobalNodeId"
}
)
},
ComparisonType = BooleanComparisonType.Equals
});
var statistics = ColumnStatistics.UpdateHistogram(newComponent.StatisticsDict[sinkNode],
newComponent.Context.GetEdgeStatistics(jointEdge));
selectivity *= statistics.Selectivity;
newComponent.StatisticsDict[sinkNode] = statistics;
if (DensityCount.ContainsKey(sinkNode.TableObjectName.ToString()))
{
DensityCount[sinkNode.TableObjectName.ToString()]++;
}
else
{
DensityCount[sinkNode.TableObjectName.ToString()] = 1;
}
}
// Leaf to component unmaterialized node
else
{
inEdges = newComponent.UnmaterializedNodeMapping[sinkNode];
var firstEdge = inEdges.First();
bool materlizedEdge = newComponent.EdgeMaterilizedDict[firstEdge];
// Leaf to materialized leaf
if (materlizedEdge)
{
joinCondition = WBooleanBinaryExpression.Conjunction(joinCondition,
new WBooleanComparisonExpression
{
FirstExpr = new WColumnReferenceExpression
{
ColumnType = ColumnType.Regular,
MultiPartIdentifier = new WMultiPartIdentifier(
new Identifier {
Value = jointEdge.EdgeAlias
},
new Identifier {
Value = "Sink"
}
),
},
SecondExpr = new WColumnReferenceExpression
{
ColumnType = ColumnType.Regular,
MultiPartIdentifier = new WMultiPartIdentifier(
new Identifier {
Value = firstEdge.EdgeAlias
},
new Identifier {
Value = "Sink"
}
)
},
ComparisonType = BooleanComparisonType.Equals
});
sinkToSinkCount++;
var statistics = ColumnStatistics.UpdateHistogram(newComponent.StatisticsDict[sinkNode],
newComponent.Context.GetEdgeStatistics(jointEdge));
selectivity *= statistics.Selectivity;
newComponent.StatisticsDict[sinkNode] = statistics;
}
// Leaf to unmaterialized leaf
else
{
ColumnStatistics compSinkNodeStatistics = null;
foreach (var inEdge in inEdges)
{
compTable = SpanTableRef(compTable, inEdge, newComponent.GetNodeRefName(inEdge.SourceNode));
newComponent.EdgeMaterilizedDict[inEdge] = true;
joinCondition = WBooleanBinaryExpression.Conjunction(joinCondition,
new WBooleanComparisonExpression
{
FirstExpr = new WColumnReferenceExpression
{
ColumnType = ColumnType.Regular,
MultiPartIdentifier = new WMultiPartIdentifier(
new Identifier {
Value = jointEdge.EdgeAlias
},
new Identifier {
Value = "Sink"
}
),
},
SecondExpr = new WColumnReferenceExpression
{
ColumnType = ColumnType.Regular,
MultiPartIdentifier = new WMultiPartIdentifier(
new Identifier {
Value = inEdge.EdgeAlias
},
new Identifier {
Value = "Sink"
}
)
},
ComparisonType = BooleanComparisonType.Equals
});
sinkToSinkCount++;
var leafToLeafStatistics = statisticsCalculator.GetLeafToLeafStatistics(jointEdge, inEdge);
selectivity *= leafToLeafStatistics.Selectivity;
compSinkNodeStatistics =
ColumnStatistics.UpdateHistogram(compSinkNodeStatistics,
newComponent.Context.GetEdgeStatistics(inEdge));
}
newComponent.StatisticsDict[sinkNode] = compSinkNodeStatistics;
}
}
}
var unmatEdges = candidateTree.UnmaterializedEdges;
foreach (var unmatEdge in unmatEdges)
{
newComponent.EdgeMaterilizedDict[unmatEdge] = false;
newComponent.Nodes.Add(unmatEdge.SinkNode);
var sinkNodeInEdges = newComponent.UnmaterializedNodeMapping.GetOrCreate(unmatEdge.SinkNode);
sinkNodeInEdges.Add(unmatEdge);
degrees *= unmatEdge.AverageDegree;
}
// Calculate Estimated Join Selectivity & Estimated Node Size
double estimatedSelectity = 1.0;
int count = 0;
bool sinkJoin = false;
foreach (var item in densityDict.Where(e => DensityCount.ContainsKey(e.Key)))
{
var density = item.Value;
var curJoinCount = DensityCount[item.Key];
var curJoinSelectitivy = Math.Pow(density, 2 - Math.Pow(2, 1 - curJoinCount));
if (!sinkJoin && ColumnStatistics.DefaultDensity < density)
{
var curSinkJoinSelectivity = Math.Pow(ColumnStatistics.DefaultDensity,
2 - Math.Pow(2, 1 - sinkToSinkCount));
estimatedSelectity *= Math.Pow(curSinkJoinSelectivity, Math.Pow(2, -count));
count += sinkToSinkCount;
sinkJoin = true;
}
estimatedSelectity *= Math.Pow(curJoinSelectitivy, Math.Pow(2, -count));
count += curJoinCount;
}
var estimatedNodeUnitSize = root.EstimatedRows *
Math.Pow(1000, candidateTree.MaterializedEdges.Count + candidateTree.UnmaterializedEdges.Count);
// Update Table Reference
newComponent.TableRef = GetPlanAndUpdateCost(candidateTree, newComponent, nodeTable, compTable, joinCondition,
degrees, selectivity, estimatedNodeUnitSize, estimatedSelectity);
return(newComponent);
}
public IEnumerable <CandidateJoinUnit> GetCandidateUnits(IEnumerable <Tuple <OneHeightTree, bool> > treeTuples, MatchComponent component)
{
foreach (var treeTuple in treeTuples)
{
var tree = treeTuple.Item1;
bool singleNode = treeTuple.Item2;
var root = tree.TreeRoot;
List <MatchEdge> jointEdges = new List <MatchEdge>();
List <MatchEdge> unpopEdges = new List <MatchEdge>();
if (singleNode)
{
unpopEdges = tree.Edges;
}
else
{
foreach (var edge in tree.Edges)
{
if (component.Nodes.Contains(edge.SinkNode))
{
jointEdges.Add(edge);
}
else
{
unpopEdges.Add(edge);
}
}
}
//var jointEdges = tree.MaterializedEdges;
//var unpopEdges = tree.UnmaterializedEdges;
int joinEdgesCount = jointEdges.Count;
int unpopEdgesCount = unpopEdges.Count;
int num = (int)Math.Pow(2, joinEdgesCount) - 1;
while (num >= 0)
{
if (joinEdgesCount > 0 && num == 0)
{
break;
}
var newJointEdges = new List <MatchEdge>();
for (int i = 0; i < joinEdgesCount; i++)
{
int index = (1 << i);
if ((num & index) != 0)
{
var edge = jointEdges[i];
newJointEdges.Add(edge);
}
}
int eNum = (int)Math.Pow(2, unpopEdgesCount) - 1;
while (eNum >= 0)
{
if (eNum == 0 && singleNode)
{
break;
}
var newUnpopEdges = new List <MatchEdge>();
for (int i = 0; i < unpopEdgesCount; i++)
{
int index = (1 << i);
if ((eNum & index) != 0)
{
newUnpopEdges.Add(unpopEdges[i]);
}
}
yield return(new CandidateJoinUnit
{
TreeRoot = root,
MaterializedEdges = newJointEdges,
UnmaterializedEdges = newUnpopEdges,
});
eNum--;
}
num--;
}
}
}
