private void ConnectPointwise(DrVertexSet source, DrVertexSet destination, ChannelConnectorType type)
{
int numberOfSrc = source.Count;
int numberOfDst = destination.Count;
if (numberOfDst == 1)
{
DrVertex ov = destination[0];
int inputs = ov.GetInputs().GetNumberOfEdges();
ov.GetInputs().GrowNumberOfEdges(inputs + numberOfSrc);
for (int i = 0; i < numberOfSrc; i++)
{
DrVertex iv = source[i];
int outputs = iv.GetOutputs().GetNumberOfEdges();
iv.GetOutputs().GrowNumberOfEdges(outputs + 1);
iv.ConnectOutput(outputs, ov, inputs + i, (DrConnectorType)type);
}
}
else if (numberOfSrc == 1)
{
DrVertex iv = source[0];
int outputs = iv.GetOutputs().GetNumberOfEdges();
iv.GetOutputs().GrowNumberOfEdges(outputs + numberOfDst);
for (int i = 0; i < numberOfDst; i++)
{
DrVertex ov = destination[i];
int inputs = ov.GetInputs().GetNumberOfEdges();
ov.GetInputs().GrowNumberOfEdges(inputs + 1);
iv.ConnectOutput(outputs + i, ov, inputs, (DrConnectorType)type);
}
}
else if (numberOfSrc == numberOfDst)
{
for (int i = 0; i < numberOfDst; i++)
{
DrVertex iv = source[i];
DrVertex ov = destination[i];
int outputs = iv.GetOutputs().GetNumberOfEdges();
int inputs = ov.GetInputs().GetNumberOfEdges();
iv.GetOutputs().GrowNumberOfEdges(outputs + 1);
ov.GetInputs().GrowNumberOfEdges(inputs + 1);
iv.ConnectOutput(outputs, ov, inputs, (DrConnectorType)type);
}
}
else
{
throw new LinqToDryadException(String.Format("Source and destination do not match in pointwise connection. Source = {0}, destination = {1}", numberOfSrc, numberOfDst));
}
}
// Add the edges to nodes
// The input ordering of each node is very important.
private void AddEdges(GraphStageInfo destInfo, Dictionary <int, GraphStageInfo> graphStageMap)
{
Vertex destVertex = destInfo.vertex;
int destId = destVertex.uniqueId;
DrVertexSet destNodes = graphStageMap[destId].members;
foreach (Predecessor p in destVertex.info.predecessors)
{
GraphStageInfo info = graphStageMap[p.uniqueId];
Vertex sourceVertex = info.vertex;
if (destVertex.type != Vertex.Type.CONCAT)
{
ChannelConnectorType channelType = ChannelConnectorType.DCT_File;
if (p.channelType == Predecessor.ChannelType.DISKFILE)
{
if (destVertex.type == Vertex.Type.OUTPUTTABLE)
{
channelType = ChannelConnectorType.DCT_Output;
}
else
{
channelType = ChannelConnectorType.DCT_File;
}
}
else if (p.channelType == Predecessor.ChannelType.MEMORYFIFO)
{
channelType = ChannelConnectorType.DCT_Fifo;
}
else if (p.channelType == Predecessor.ChannelType.TCPPIPE)
{
channelType = ChannelConnectorType.DCT_Pipe;
}
else
{
string msg = String.Format("Unknown channel type {0}", p.channelType);
throw new LinqToDryadException(msg);
}
DrVertexSet sourceNodes = info.members;
switch (p.connectionOperator)
{
case Predecessor.ConnectionOperator.CROSSPRODUCT:
ConnectCrossProduct(sourceNodes, destNodes, channelType);
break;
case Predecessor.ConnectionOperator.POINTWISE:
ConnectPointwise(sourceNodes, destNodes, channelType);
break;
default:
break;
}
}
}
}
private DrVertexSet CreateVertexSet(DrGraph graph, DrVertex prototype, int copies)
{
DrVertexSet result = new DrVertexSet();
for (int i = 0; i < copies; i++)
{
DrVertex v = prototype.MakeCopy(i);
result.Add(v);
}
return(result);
}
private DrVertexSet CreateVertexSet(DrGraph graph, DrInputStreamManager inputStream)
{
DrVertexSet result = new DrVertexSet();
List <DrStorageVertex> vertices = inputStream.GetVertices();
for (int i = 0; i < vertices.Count; i++)
{
DrVertex v = vertices[i];
result.Add(v);
}
return(result);
}
private DrVertexSet CreateVertexSet(DrGraph graph, DrOutputStreamManager outputStream, int parts)
{
DrVertexSet result = new DrVertexSet();
outputStream.SetNumberOfParts(parts);
List <DrOutputVertex> vertices = outputStream.GetVertices();
for (int i = 0; i < vertices.Count; i++)
{
DrVertex v = vertices[i];
result.Add(v);
}
return(result);
}
private void ConnectCrossProduct(DrVertexSet source, DrVertexSet destination, ChannelConnectorType type)
{
int numberOfSrc = source.Count;
int numberOfDst = destination.Count;
// Allocate ports
for (int i = 0; i < numberOfSrc; i++)
{
source[i].GetOutputs().SetNumberOfEdges(numberOfDst);
}
// Connect them up
for (int destinationNumber = 0; destinationNumber < numberOfDst; destinationNumber++)
{
DrVertex dv = destination[destinationNumber];
int inputs = dv.GetInputs().GetNumberOfEdges();
dv.GetInputs().GrowNumberOfEdges(inputs + numberOfSrc);
for (int sourceNumber = 0; sourceNumber < numberOfSrc; sourceNumber++)
{
source[sourceNumber].ConnectOutput(destinationNumber, dv, inputs + sourceNumber, (DrConnectorType)type);
}
}
}
public GraphStageInfo(DrVertexSet m, Vertex v, DrStageManager mgr)
{
members = m;
vertex = v;
stageManager = mgr;
}
private void CreateVertexSet(Vertex v, DryadLINQApp app, Query query, Dictionary <int, GraphStageInfo> graphStageMap)
{
SortedDictionary <int, Vertex> queryPlan = query.queryPlan;
DrVertexSet nodes = null;
DrStageManager newManager = null;
app.GetGraph().GetParameters();
DrGraphParameters parameters = app.GetGraph().GetParameters();
string stdVertexName = "MW";
if (v.type == Vertex.Type.INPUTTABLE)
{
DrInputStreamManager input = CreateInputNode(app, v.info, v.name);
newManager = input.GetStageManager();
nodes = CreateVertexSet(app.GetGraph(), input);
}
else if (v.type == Vertex.Type.OUTPUTTABLE)
{
DrOutputStreamManager output = CreateOutputNode(app, v.info, v.name);
newManager = output.GetStageManager();
nodes = CreateVertexSet(app.GetGraph(), output, v.partitions);
}
else if (v.type == Vertex.Type.CONCAT)
{
newManager = new DrManagerBase(app.GetGraph(), v.name);
// the set of nodes in a concat is just the set of nodes in the predecessor stages concatenated
nodes = new DrVertexSet();
foreach (Predecessor p in v.info.predecessors)
{
GraphStageInfo value = null;
if (graphStageMap.TryGetValue(p.uniqueId, out value))
{
nodes.InsertRange(nodes.Count, value.members);
}
else
{
throw new LinqToDryadException(String.Format("Concat: Failed to find predecessor {0} in graph stage map", p.uniqueId));
}
}
}
else
{
newManager = new DrManagerBase(app.GetGraph(), v.name);
DrVertex vertex;
if (v.type == Vertex.Type.TEE)
{
DrTeeVertex teeVertex = new DrTeeVertex(newManager);
vertex = teeVertex;
}
else
{
DrActiveVertex activeVertex = new DrActiveVertex(newManager, parameters.m_defaultProcessTemplate, parameters.m_defaultVertexTemplate);
activeVertex.AddArgument(stdVertexName);
activeVertex.AddArgument(v.info.assemblyName);
activeVertex.AddArgument(v.info.className);
activeVertex.AddArgument(v.info.methodName);
vertex = activeVertex;
}
nodes = CreateVertexSet(app.GetGraph(), vertex, v.partitions);
if (v.machines != null && v.machines.Length != 0 && v.type != Vertex.Type.TEE)
{
for (int i = 0; i < v.partitions; i++)
{
DrResource r = app.GetUniverse().LookUpResource(v.machines[i]);
if (r != null)
{
DrVertex baseVertex = nodes[i];
DrActiveVertex activeVertex = baseVertex as DrActiveVertex;
activeVertex.GetAffinity().SetHardConstraint(true);
activeVertex.GetAffinity().AddLocality(r);
}
}
}
FileStream mapfile = app.GetIdentityMapFile();
if (mapfile != null)
{
for (int i = 0; i < v.partitions; i++)
{
DrVertex jmv = nodes[i];
string message = String.Format("Mapping {0}[{1}] to {2}\n", v.uniqueId, i, jmv.GetId());
System.Text.ASCIIEncoding encoding = new System.Text.ASCIIEncoding();
mapfile.Seek(0, SeekOrigin.End);
mapfile.Write(encoding.GetBytes(message), 0, message.Length);
}
}
}
graphStageMap.Add(v.uniqueId, new GraphStageInfo(nodes, v, newManager));
}
