private DrVertexSet CreateVertexSet(DrGraph graph, DrOutputStreamManager outputStream, int parts) { DrVertexSet result = new DrVertexSet(); outputStream.SetNumberOfParts(parts); List <DrOutputVertex> vertices = outputStream.GetVertices(); for (int i = 0; i < vertices.Count; i++) { DrVertex v = vertices[i]; result.Add(v); } return(result); }
private void CreateVertexSet(Vertex v, DryadLINQApp app, Query query, Dictionary <int, GraphStageInfo> graphStageMap) { SortedDictionary <int, Vertex> queryPlan = query.queryPlan; DrVertexSet nodes = null; DrStageManager newManager = null; app.GetGraph().GetParameters(); DrGraphParameters parameters = app.GetGraph().GetParameters(); string stdVertexName = "MW"; if (v.type == Vertex.Type.INPUTTABLE) { DrInputStreamManager input = CreateInputNode(app, v.info, v.name); newManager = input.GetStageManager(); nodes = CreateVertexSet(app.GetGraph(), input); } else if (v.type == Vertex.Type.OUTPUTTABLE) { DrOutputStreamManager output = CreateOutputNode(app, v.info, v.name); newManager = output.GetStageManager(); nodes = CreateVertexSet(app.GetGraph(), output, v.partitions); } else if (v.type == Vertex.Type.CONCAT) { newManager = new DrManagerBase(app.GetGraph(), v.name); // the set of nodes in a concat is just the set of nodes in the predecessor stages concatenated nodes = new DrVertexSet(); foreach (Predecessor p in v.info.predecessors) { GraphStageInfo value = null; if (graphStageMap.TryGetValue(p.uniqueId, out value)) { nodes.InsertRange(nodes.Count, value.members); } else { throw new LinqToDryadException(String.Format("Concat: Failed to find predecessor {0} in graph stage map", p.uniqueId)); } } } else { newManager = new DrManagerBase(app.GetGraph(), v.name); DrVertex vertex; if (v.type == Vertex.Type.TEE) { DrTeeVertex teeVertex = new DrTeeVertex(newManager); vertex = teeVertex; } else { DrActiveVertex activeVertex = new DrActiveVertex(newManager, parameters.m_defaultProcessTemplate, parameters.m_defaultVertexTemplate); activeVertex.AddArgument(stdVertexName); activeVertex.AddArgument(v.info.assemblyName); activeVertex.AddArgument(v.info.className); activeVertex.AddArgument(v.info.methodName); vertex = activeVertex; } nodes = CreateVertexSet(app.GetGraph(), vertex, v.partitions); if (v.machines != null && v.machines.Length != 0 && v.type != Vertex.Type.TEE) { for (int i = 0; i < v.partitions; i++) { DrResource r = app.GetUniverse().LookUpResource(v.machines[i]); if (r != null) { DrVertex baseVertex = nodes[i]; DrActiveVertex activeVertex = baseVertex as DrActiveVertex; activeVertex.GetAffinity().SetHardConstraint(true); activeVertex.GetAffinity().AddLocality(r); } } } FileStream mapfile = app.GetIdentityMapFile(); if (mapfile != null) { for (int i = 0; i < v.partitions; i++) { DrVertex jmv = nodes[i]; string message = String.Format("Mapping {0}[{1}] to {2}\n", v.uniqueId, i, jmv.GetId()); System.Text.ASCIIEncoding encoding = new System.Text.ASCIIEncoding(); mapfile.Seek(0, SeekOrigin.End); mapfile.Write(encoding.GetBytes(message), 0, message.Length); } } } graphStageMap.Add(v.uniqueId, new GraphStageInfo(nodes, v, newManager)); }
private DrOutputStreamManager CreateOutputNode(DryadLINQApp app, VertexInfo info, string outputName) { DryadLogger.LogInformation("methodEntry: " + outputName); DrOutputStreamManager s; if (info.ioType == VertexInfo.IOType.PARTITIONEDFILE) { DrPartitionOutputStream output = new DrPartitionOutputStream(); Uri sourceUri = new Uri(info.sources[0]); string sourcePath = sourceUri.AbsolutePath; if (!String.IsNullOrEmpty(sourceUri.Host)) { sourcePath = @"\\" + sourceUri.Host + sourcePath; } int err = output.Open(sourcePath, info.partitionUncPath); if (!SUCCEEDED(err)) { string msg = String.Format("Could not open output fileset {0}", sourcePath); throw new LinqToDryadException(msg, err); } DrManagerBase outputStage = new DrManagerBase(app.GetGraph(), outputName); DrOutputStreamManager outputManager = new DrOutputStreamManager(output, outputStage); app.GetGraph().AddPartitionGenerator(outputManager); s = outputManager; } //else if ( info.ioType == VertexInfo.IOType.STREAM ) //{ // DrDscOutputStream output = new DrDscOutputStream(info.compressionScheme, info.isTemporary); // int err = 0; // if (info.recordType == "") // { // err = output.Open(info.sources[0], info.partitionUncPath); // } // else // { // err = output.OpenWithRecordType(info.sources[0], info.partitionUncPath, info.recordType); // } // if (!SUCCEEDED(err)) // { // string msg = String.Format("Could not open DSC output fileset {0}", info.sources[0]); // throw new LinqToDryadException(msg, err); // } // DrManagerBase outputStage = new DrManagerBase(app.GetGraph(), outputName); // DrOutputStreamManager outputManager = new DrOutputStreamManager(output, outputStage); // app.GetGraph().AddPartitionGenerator(outputManager); // s = outputManager; //} else if (info.ioType == VertexInfo.IOType.HDFS_STREAM) { DrHdfsOutputStream output = new DrHdfsOutputStream(); int err = output.Open(info.sources[0]); if (!SUCCEEDED(err)) { string msg = String.Format("Could not open HDFS output fileset {0}: {1}", info.sources[0], output.GetError()); throw new LinqToDryadException(msg, err); } DrManagerBase outputStage = new DrManagerBase(app.GetGraph(), outputName); DrOutputStreamManager outputManager = new DrOutputStreamManager(output, outputStage); app.GetGraph().AddPartitionGenerator(outputManager); s = outputManager; } else if (info.ioType == VertexInfo.IOType.AZUREBLOB) { DrAzureOutputStream output = new DrAzureOutputStream(); try { output.Open(info.sources[0]); } catch (Exception e) { string msg = String.Format("Could not open Azure output fileset {0}: {1}", info.sources[0], e.ToString()); throw new LinqToDryadException(msg); } DrManagerBase outputStage = new DrManagerBase(app.GetGraph(), outputName); DrOutputStreamManager outputManager = new DrOutputStreamManager(output, outputStage); app.GetGraph().AddPartitionGenerator(outputManager); s = outputManager; } else { string msg = String.Format("Unknown output type {0}", info.ioType); throw new LinqToDryadException(msg); } return(s); }
private DrOutputStreamManager CreateOutputNode(DryadLINQApp app, VertexInfo info, string outputName) { DrOutputStreamManager s; if (info.ioType == VertexInfo.IOType.PARTITIONEDFILE) { DrPartitionOutputStream output = new DrPartitionOutputStream(); int err = output.Open(info.sources[0], info.partitionUncPath); if (!SUCCEEDED(err)) { string msg = String.Format("Could not open DSC output fileset {0}", info.sources[0]); throw new LinqToDryadException(msg, err); } DrManagerBase outputStage = new DrManagerBase(app.GetGraph(), outputName); DrOutputStreamManager outputManager = new DrOutputStreamManager(output, outputStage); app.GetGraph().AddPartitionGenerator(outputManager); s = outputManager; } //else if ( info.ioType == VertexInfo.IOType.STREAM ) //{ // DrDscOutputStream output = new DrDscOutputStream(info.compressionScheme, info.isTemporary); // int err = 0; // if (info.recordType == "") // { // err = output.Open(info.sources[0], info.partitionUncPath); // } // else // { // err = output.OpenWithRecordType(info.sources[0], info.partitionUncPath, info.recordType); // } // if (!SUCCEEDED(err)) // { // string msg = String.Format("Could not open DSC output fileset {0}", info.sources[0]); // throw new LinqToDryadException(msg, err); // } // DrManagerBase outputStage = new DrManagerBase(app.GetGraph(), outputName); // DrOutputStreamManager outputManager = new DrOutputStreamManager(output, outputStage); // app.GetGraph().AddPartitionGenerator(outputManager); // s = outputManager; //} else if (info.ioType == VertexInfo.IOType.HDFS_STREAM) { DrHdfsOutputStream output = new DrHdfsOutputStream(); int err = output.Open(info.sources[0]); if (!SUCCEEDED(err)) { string msg = String.Format("Could not open HDFS output fileset {0}", info.sources[0]); throw new LinqToDryadException(msg, err); } DrManagerBase outputStage = new DrManagerBase(app.GetGraph(), outputName); DrOutputStreamManager outputManager = new DrOutputStreamManager(output, outputStage); app.GetGraph().AddPartitionGenerator(outputManager); s = outputManager; } else { string msg = String.Format("Unknown output type {0}", info.ioType); throw new LinqToDryadException(msg); } return(s); }