internal virtual string Visit(DryadOutputNode node,
CodeMemberMethod vertexMethod,
string[] readerNames,
string[] writerNames)
{
return node.AddVertexCode(vertexMethod, readerNames, writerNames);
}
// Phase 1 of the query optimization
internal void GenerateQueryPlanPhase1()
{
if (this.m_queryPlan1 != null) return;
// Apply some simple rewrite rules
SimpleRewriter rewriter = new SimpleRewriter(this.m_exprNodeInfoMap.Values.ToList());
rewriter.Rewrite();
// Generate the query plan of phase1
var referencedNodes = this.m_referencedQueryMap.Values;
this.m_queryPlan1 = new DryadQueryNode[this.m_queryExprs.Length + referencedNodes.Count];
for (int i = 0; i < this.m_queryExprs.Length; i++)
{
this.m_queryPlan1[i] = this.Visit(this.m_queryNodeInfos[i].children[0].child);
}
int idx = this.m_queryExprs.Length;
foreach (QueryNodeInfo nodeInfo in referencedNodes)
{
// Add a Tee'd Merge
this.m_queryPlan1[idx] = this.Visit(nodeInfo.children[0].child);
DryadQueryNode mergeNode = new DryadMergeNode(true, false, nodeInfo.queryExpression,
this.m_queryPlan1[idx]);
this.m_queryPlan1[idx] = new DryadTeeNode(mergeNode.OutputTypes[0], true,
mergeNode.QueryExpression, mergeNode);
nodeInfo.queryNode = this.m_queryPlan1[idx];
idx++;
}
// Finally, add the output nodes.
Dictionary<DryadQueryNode, int> forkCounts = new Dictionary<DryadQueryNode, int>();
for (int i = 0; i < this.m_queryExprs.Length; i++)
{
DryadQueryNode queryNode = this.m_queryPlan1[i];
int cnt;
if (!forkCounts.TryGetValue(queryNode, out cnt))
{
cnt = queryNode.Parents.Count;
}
forkCounts[queryNode] = cnt + 1;
}
for (int i = 0; i < this.m_queryExprs.Length; i++)
{
HpcClientSideLog.Add("Query " + i + " Output: " + this.m_outputDatapaths[i]);
DryadQueryNode queryNode = this.m_queryPlan1[i];
if (TypeSystem.IsAnonymousType(queryNode.OutputTypes[0]))
{
throw new DryadLinqException(HpcLinqErrorCode.OutputTypeCannotBeAnonymous,
SR.OutputTypeCannotBeAnonymous);
}
// Add dummy Apply to make Dryad happy (it doesn't like to hook inputs straight to outputs)
if ((queryNode is DryadInputNode) || (forkCounts[queryNode] > 1))
{
// Add a dummy Apply
Type paramType = typeof(IEnumerable<>).MakeGenericType(queryNode.OutputTypes[0]);
ParameterExpression param = Expression.Parameter(paramType, "x");
Type type = typeof(Func<,>).MakeGenericType(paramType, paramType);
LambdaExpression applyExpr = Expression.Lambda(type, param, param);
DryadQueryNode applyNode = new DryadApplyNode(applyExpr, this.m_queryExprs[i], queryNode);
applyNode.OutputDataSetInfo = queryNode.OutputDataSetInfo;
queryNode = applyNode;
}
if (queryNode is DryadConcatNode)
{
// Again, we add dummy Apply in certain cases to make Dryad happy
((DryadConcatNode)queryNode).FixInputs();
}
// Add the output node
DscCompressionScheme outputScheme = this.m_context.Configuration.OutputDataCompressionScheme;
DryadOutputNode outputNode = new DryadOutputNode(this.m_context,
this.m_outputDatapaths[i],
this.m_isTempOutput[i],
outputScheme,
this.m_queryExprs[i],
queryNode);
this.m_queryPlan1[i] = outputNode;
if (this.m_outputUriMap.ContainsKey(this.m_outputDatapaths[i].ToLower()))
{
throw new DryadLinqException(HpcLinqErrorCode.MultipleOutputsWithSameDscUri,
String.Format(SR.MultipleOutputsWithSameDscUri, this.m_outputDatapaths[i]));
}
this.m_outputUriMap.Add(this.m_outputDatapaths[i].ToLower(), outputNode);
this.m_outputTypes[i] = this.m_queryPlan1[i].OutputTypes[0];
// Remove useless Tees to make Dryad happy
if ((queryNode is DryadTeeNode) && (forkCounts[queryNode] == 1))
{
DryadQueryNode teeChild = queryNode.Children[0];
teeChild.UpdateParent(queryNode, outputNode);
outputNode.UpdateChildren(queryNode, teeChild);
}
}
}
// create DryadLinqMetaData from a query OutputNode
internal static DryadLinqMetaData FromOutputNode(HpcLinqContext context, DryadOutputNode node)
{
DryadLinqMetaData metaData = new DryadLinqMetaData();
if (! (DataPath.IsDsc(node.MetaDataUri) || DataPath.IsHdfs(node.MetaDataUri)) )
{
throw new InvalidOperationException();
}
metaData.m_context = context;
metaData.m_dscStreamName = node.MetaDataUri;
metaData.m_elemType = node.OutputTypes[0];
metaData.m_compressionScheme = node.OutputCompressionScheme;
//metaData.m_version = context.ClientVersion;
//metaData.InitializeFlags();
//metaData.m_fp = 0UL;
//metaData.m_dataSetInfo = node.OutputDataSetInfo;
return metaData;
}