internal ProjectionExpression(SelectExpression source, Expression projector, LambdaExpression aggregator)
: base(PigExpressionType.Projection, aggregator != null ? aggregator.Body.Type : typeof(IEnumerable<>).MakeGenericType(projector.Type))
{
Source = source;
Projector = projector;
Aggregator = aggregator;
}
protected override Expression VisitSelect(SelectExpression node)
{
// visit column projection first
ReadOnlyCollection<ColumnDeclaration> columns = node.Columns;
List<ColumnDeclaration> alternate = null;
for (Int32 i = 0, n = node.Columns.Count; i < n; i++)
{
ColumnDeclaration decl = node.Columns[i];
if (IsColumnUsed(node.Alias, decl.Name))
{
Expression expr = Visit(decl.Expression);
if (expr != decl.Expression)
{
decl = new ColumnDeclaration(decl.Name, expr);
}
}
else
{
decl = null; // null means it gets omitted
}
if (decl != node.Columns[i] && alternate == null)
{
alternate = new List<ColumnDeclaration>();
for (Int32 j = 0; j < i; j++)
{
alternate.Add(node.Columns[j]);
}
}
if (decl != null && alternate != null)
{
alternate.Add(decl);
}
}
if (alternate != null)
{
columns = alternate.AsReadOnly();
}
Expression take = Visit(node.Take);
ReadOnlyCollection<Expression> groupbys = VisitExpressionList(node.GroupBy);
ReadOnlyCollection<OrderByExpression> orderbys = VisitOrderBy(node.OrderBy);
Expression where = Visit(node.Where);
Expression from = Visit(node.From);
ClearColumnsUsed(node.Alias);
if (columns != node.Columns
|| take != node.Take
|| orderbys != node.OrderBy
|| groupbys != node.GroupBy
|| where != node.Where
|| from != node.From)
{
node = new SelectExpression(node.Alias, columns, from, where, orderbys, groupbys, take);
}
return node;
}
protected override Expression VisitSelect(SelectExpression node)
{
if (_selectsToRemove.Contains(node))
{
return Visit(node.From);
}
return base.VisitSelect(node);
}
private static Boolean ProjectionIsNameMapOnly(SelectExpression select)
{
var fromSelect = select.From as SelectExpression;
if (fromSelect == null || select.Columns.Count != fromSelect.Columns.Count)
return false;
// test that all _columns in 'select' are refering to _columns in the same position
// in 'fromSelect'.
for (Int32 i = 0, n = select.Columns.Count; i < n; i++)
{
var col = select.Columns[i].Expression as ColumnExpression;
if (col == null || !(col.Name == fromSelect.Columns[i].Name))
return false;
}
return true;
}
// protected override methods
protected override Expression VisitSelect(SelectExpression node)
{
node = (SelectExpression) base.VisitSelect(node);
if (_lookup.Contains(node.Alias))
{
var aggColumns = new List<ColumnDeclaration>(node.Columns);
foreach (AggregateSubqueryExpression ae in _lookup[node.Alias])
{
String name = "agg" + aggColumns.Count;
var cd = new ColumnDeclaration(name, ae.AggregateInGroupSelect);
_map.Add(ae, new ColumnExpression(ae.Type, ae.GroupByAlias, name));
aggColumns.Add(cd);
}
return new SelectExpression(node.Alias, aggColumns, node.From, node.Where, node.OrderBy, node.GroupBy,
node.Take);
}
return node;
}
// protected override methods
protected override Expression VisitSelect(SelectExpression node)
{
node = (SelectExpression) base.VisitSelect(node);
// first remove all purely redundant subqueries
List<SelectExpression> redundant = RedundantSubqueryGatherer.Gather(node.From);
if (redundant != null)
{
node = SubqueryRemover.Remove(node, redundant);
}
// next attempt to merge subqueries that would have been removed by the above
// logic except for the existence of a where clause
//while (CanMergeWithFrom(node))
//{
// SelectExpression fromSelect = (SelectExpression)node.From;
// // remove the redundant subquery
// node = SubqueryRemover.Remove(node, fromSelect);
// // merge where expressions
// Expression where = node.Where;
// if (fromSelect.Where != null)
// {
// if (where != null)
// {
// where = Expression.And(fromSelect.Where, where);
// }
// else
// {
// where = fromSelect.Where;
// }
// }
// if (where != node.Where)
// {
// node = new SelectExpression(node.Type, node.Alias, node.Columns, node.From, where, node.OrderBy, node.GroupBy);
// }
//}
return node;
}
protected virtual Expression VisitSelect(SelectExpression node)
{
Expression from = VisitSource(node.From);
Expression where = Visit(node.Where);
ReadOnlyCollection<ColumnDeclaration> columns = VisitColumnDeclarations(node.Columns);
ReadOnlyCollection<OrderByExpression> orderBy = VisitOrderBy(node.OrderBy);
ReadOnlyCollection<Expression> groupBy = VisitExpressionList(node.GroupBy);
Expression take = Visit(node.Take);
if (from != node.From
|| where != node.Where
|| columns != node.Columns
|| orderBy != node.OrderBy
|| groupBy != node.GroupBy
|| take != node.Take)
{
return new SelectExpression(node.Alias, columns, from, where, orderBy, groupBy, take);
}
return node;
}
// private static methods
private static Boolean IsRedudantSubquery(SelectExpression select)
{
return (select.From is SelectExpression || select.From is SourceExpression)
&& (ProjectionIsSimple(select) || ProjectionIsNameMapOnly(select))
&& (select.Where == null)
&& (select.Take == null)
&& (select.OrderBy == null || select.OrderBy.Count == 0)
&& (select.GroupBy == null || select.GroupBy.Count == 0);
}
// protected override methods
protected override Expression VisitSelect(SelectExpression select)
{
if (IsRedudantSubquery(select))
{
if (_redundant == null)
{
_redundant = new List<SelectExpression>();
}
_redundant.Add(select);
}
return select;
}
internal ScalarExpression(Type type, SelectExpression select)
: base(PigExpressionType.Scalar, type, select)
{
}
// constructors
internal SubqueryExpression(PigExpressionType eType, Type type, SelectExpression select)
: base(eType, type)
{
Select = select;
}
// private methods
private Expression BindAggregate(Expression source, String aggName, MethodInfo method, LambdaExpression argument,
Boolean isRoot)
{
Type returnType = method.ReturnType;
Boolean hasPredicateArg = HasPredicateArg(aggName);
Boolean isDistinct = false;
Boolean argumentWasPredicate = false;
// check for distinct
var mcs = source as MethodCallExpression;
if (mcs != null && !hasPredicateArg && argument == null)
{
if (mcs.Method.Name == "Distinct" && mcs.Arguments.Count == 1 &&
(mcs.Method.DeclaringType == typeof (Queryable) || mcs.Method.DeclaringType == typeof (Enumerable)))
{
source = mcs.Arguments[0];
isDistinct = true;
}
}
if (argument != null && hasPredicateArg)
{
// convert query.Count(predicate) into query.Where(predicate).Count()
source = Expression.Call(typeof (Queryable), "Where", method.GetGenericArguments(), source, argument);
argument = null;
argumentWasPredicate = true;
}
ProjectionExpression projection = VisitSequence(source);
Expression argExpr = null;
if (argument != null)
{
_map[argument.Parameters[0]] = projection.Projector;
argExpr = Visit(argument.Body);
}
else if (!hasPredicateArg)
{
argExpr = projection.Projector;
}
SourceAlias alias = GetNextAlias();
Expression aggExpr = new AggregateExpression(returnType, aggName, argExpr, isDistinct);
var select = new SelectExpression(alias, new[] {new ColumnDeclaration("", aggExpr)}, projection.Source, null);
if (isRoot)
{
ParameterExpression p = Expression.Parameter(typeof (IEnumerable<>).MakeGenericType(aggExpr.Type),
"node");
LambdaExpression gator =
Expression.Lambda(Expression.Call(typeof (Enumerable), "Single", new[] {returnType}, p), p);
return new ProjectionExpression(select, new ColumnExpression(returnType, alias, ""), gator);
}
var subquery = new ScalarExpression(returnType, select);
// if we can find the corresponding group-info we can build a special AggregateSubquery node that will enable us to
// optimize the node expression later using AggregateRewriter
GroupByInfo info;
if (!argumentWasPredicate && _groupByMap.TryGetValue(projection, out info))
{
// use the element expression from the group-by info to rebind the argument so the resulting expression is one that
// would be legal to add to the _columns in the select expression that has the corresponding group-by clause.
if (argument != null)
{
_map[argument.Parameters[0]] = info.Element;
argExpr = Visit(argument.Body);
}
else if (!hasPredicateArg)
{
argExpr = info.Element;
}
aggExpr = new AggregateExpression(returnType, aggName, argExpr, isDistinct);
// check for easy to optimize case. If the projection that our node is based on is really the 'group' argument from
// the query.GroupBy(xxx, (key, group) => yyy) method then whatever expression we return here will automatically
// become part of the select expression that has the group-by clause, so just return the simple node expression.
if (projection == _currentGroupElement)
{
return aggExpr;
}
return new AggregateSubqueryExpression(info.Alias, aggExpr, subquery);
}
return subquery;
}
// internal static methods
internal static SelectExpression Remove(SelectExpression outerSelect,
IEnumerable<SelectExpression> selectsToRemove)
{
return (SelectExpression) new SubqueryRemover(selectsToRemove).Visit(outerSelect);
}
// constructors
internal ProjectionExpression(SelectExpression source, Expression projector)
: this(source, projector, null)
{
}
// protected methods
protected override Expression VisitSelect(SelectExpression select)
{
_aliases.Add(select.Alias);
return select;
}
protected override Expression VisitSelect(SelectExpression select)
{
Boolean saveIsOuterMostSelect = _isOuterMostSelect;
try
{
_isOuterMostSelect = false;
select = (SelectExpression) base.VisitSelect(select);
Boolean hasOrderBy = select.OrderBy != null && select.OrderBy.Count > 0;
Boolean hasGroupBy = select.GroupBy != null && select.GroupBy.Count > 0;
Boolean canHaveOrderBy = saveIsOuterMostSelect || select.Take != null;
Boolean canReceiveOrderings = canHaveOrderBy && !hasGroupBy;
if (hasOrderBy)
{
PrependOrderings(select.OrderBy);
}
IEnumerable<OrderByExpression> orderings = null;
if (canReceiveOrderings)
{
orderings = _gatheredOrderings;
}
else if (canHaveOrderBy)
{
orderings = select.OrderBy;
}
Boolean canPassOnOrderings = !saveIsOuterMostSelect && !hasGroupBy;
ReadOnlyCollection<ColumnDeclaration> columns = select.Columns;
if (_gatheredOrderings != null)
{
if (canPassOnOrderings)
{
HashSet<SourceAlias> producedAliases = AliasesProduced.Gather(select.From);
// reproject order expressions using this select's alias so the outer select will have properly formed expressions
BindResult project = RebindOrderings(_gatheredOrderings, select.Alias, producedAliases,
select.Columns);
_gatheredOrderings = null;
PrependOrderings(project.Orderings);
columns = project.Columns;
}
else
{
_gatheredOrderings = null;
}
}
if (orderings != select.OrderBy || columns != select.Columns)
{
select = new SelectExpression(select.Alias, columns, select.From, select.Where, orderings,
select.GroupBy, select.Take);
}
return select;
}
finally
{
_isOuterMostSelect = saveIsOuterMostSelect;
}
}
// private static methods
private static Boolean ProjectionIsSimple(SelectExpression select)
{
foreach (ColumnDeclaration decl in select.Columns)
{
var col = decl.Expression as ColumnExpression;
if (col == null || decl.Name != col.Name)
{
return false;
}
}
return true;
}