private DrVertexSet CreateVertexSet(DrGraph graph, DrInputStreamManager inputStream) { DrVertexSet result = new DrVertexSet(); List <DrStorageVertex> vertices = inputStream.GetVertices(); for (int i = 0; i < vertices.Count; i++) { DrVertex v = vertices[i]; result.Add(v); } return(result); }
private DrInputStreamManager CreateInputNode(DryadLINQApp app, VertexInfo info, string inputName) { DrInputStreamManager s; int err = 0; DryadLogger.LogInformation("methodEntry: " + inputName); if (info.ioType == VertexInfo.IOType.PARTITIONEDFILE) { DrPartitionInputStream input = new DrPartitionInputStream(); err = input.Open(app.GetUniverse(), new Uri(info.sources[0]).AbsolutePath); if (!SUCCEEDED(err)) { string msg = String.Format("Could not read Partitioned file input {0}", info.sources[0]); throw new LinqToDryadException(msg, err); } DrManagerBase inputStage = new DrManagerBase(app.GetGraph(), inputName); DrInputStreamManager inputManager = new DrInputStreamManager(input, inputStage); s = inputManager; } //else if ( info.ioType == VertexInfo.IOType.STREAM ) //{ // DrDscInputStream input = new DrDscInputStream(); // DryadLogger.LogInformation("Create input node", "Opening DSC input fileset"); // err = input.Open(app.GetUniverse(), info.sources[0]); // if (!SUCCEEDED(err)) // { // string msg = String.Format("Could not read DSC input fileset {0}", info.sources[0]); // throw new LinqToDryadException(msg, err); // } // DryadLogger.LogInformation("Create input node", "Opened DSC input fileset"); // DrManagerBase inputStage = new DrManagerBase(app.GetGraph(), inputName); // DrInputStreamManager inputManager = new DrInputStreamManager(input, inputStage); // s = inputManager; //} else if (info.ioType == VertexInfo.IOType.HDFS_STREAM) { DrHdfsInputStream input = new DrHdfsInputStream(); DryadLogger.LogInformation("Create input node", "Opening HDFS input fileset"); Uri srcUri = new Uri(info.sources[0]); err = input.Open(app.GetUniverse(), srcUri.GetLeftPart(UriPartial.Path), info.recordType); if (!SUCCEEDED(err)) { string msg = String.Format("Could not read HDFS input fileset {0}: {1}", info.sources[0], input.GetError()); throw new LinqToDryadException(msg, err); } DryadLogger.LogInformation("Create input node", "Opened HDFS input fileset"); DrManagerBase inputStage = new DrManagerBase(app.GetGraph(), inputName); DrInputStreamManager inputManager = new DrInputStreamManager(input, inputStage); s = inputManager; } else if (info.ioType == VertexInfo.IOType.AZUREBLOB) { DrAzureInputStream input = new DrAzureInputStream(); DryadLogger.LogInformation("Create input node", "Opening Azure input fileset"); try { input.Open(info.sources[0]); } catch (Exception e) { string msg = String.Format("Could not read Azure input fileset {0}: {1}", info.sources[0], e.ToString()); throw new LinqToDryadException(msg); } DryadLogger.LogInformation("Create input node", "Opened Azure input fileset"); DrManagerBase inputStage = new DrManagerBase(app.GetGraph(), inputName); DrInputStreamManager inputManager = new DrInputStreamManager(input, inputStage); s = inputManager; } else { string msg = String.Format("Unknown input type {0}", info.ioType); throw new LinqToDryadException(msg); } DryadLogger.LogInformation("methodExit"); return(s); }
private void CreateVertexSet(Vertex v, DryadLINQApp app, Query query, Dictionary <int, GraphStageInfo> graphStageMap) { SortedDictionary <int, Vertex> queryPlan = query.queryPlan; DrVertexSet nodes = null; DrStageManager newManager = null; app.GetGraph().GetParameters(); DrGraphParameters parameters = app.GetGraph().GetParameters(); string stdVertexName = "MW"; if (v.type == Vertex.Type.INPUTTABLE) { DrInputStreamManager input = CreateInputNode(app, v.info, v.name); newManager = input.GetStageManager(); nodes = CreateVertexSet(app.GetGraph(), input); } else if (v.type == Vertex.Type.OUTPUTTABLE) { DrOutputStreamManager output = CreateOutputNode(app, v.info, v.name); newManager = output.GetStageManager(); nodes = CreateVertexSet(app.GetGraph(), output, v.partitions); } else if (v.type == Vertex.Type.CONCAT) { newManager = new DrManagerBase(app.GetGraph(), v.name); // the set of nodes in a concat is just the set of nodes in the predecessor stages concatenated nodes = new DrVertexSet(); foreach (Predecessor p in v.info.predecessors) { GraphStageInfo value = null; if (graphStageMap.TryGetValue(p.uniqueId, out value)) { nodes.InsertRange(nodes.Count, value.members); } else { throw new LinqToDryadException(String.Format("Concat: Failed to find predecessor {0} in graph stage map", p.uniqueId)); } } } else { newManager = new DrManagerBase(app.GetGraph(), v.name); DrVertex vertex; if (v.type == Vertex.Type.TEE) { DrTeeVertex teeVertex = new DrTeeVertex(newManager); vertex = teeVertex; } else { DrActiveVertex activeVertex = new DrActiveVertex(newManager, parameters.m_defaultProcessTemplate, parameters.m_defaultVertexTemplate); activeVertex.AddArgument(stdVertexName); activeVertex.AddArgument(v.info.assemblyName); activeVertex.AddArgument(v.info.className); activeVertex.AddArgument(v.info.methodName); vertex = activeVertex; } nodes = CreateVertexSet(app.GetGraph(), vertex, v.partitions); if (v.machines != null && v.machines.Length != 0 && v.type != Vertex.Type.TEE) { for (int i = 0; i < v.partitions; i++) { DrResource r = app.GetUniverse().LookUpResource(v.machines[i]); if (r != null) { DrVertex baseVertex = nodes[i]; DrActiveVertex activeVertex = baseVertex as DrActiveVertex; activeVertex.GetAffinity().SetHardConstraint(true); activeVertex.GetAffinity().AddLocality(r); } } } FileStream mapfile = app.GetIdentityMapFile(); if (mapfile != null) { for (int i = 0; i < v.partitions; i++) { DrVertex jmv = nodes[i]; string message = String.Format("Mapping {0}[{1}] to {2}\n", v.uniqueId, i, jmv.GetId()); System.Text.ASCIIEncoding encoding = new System.Text.ASCIIEncoding(); mapfile.Seek(0, SeekOrigin.End); mapfile.Write(encoding.GetBytes(message), 0, message.Length); } } } graphStageMap.Add(v.uniqueId, new GraphStageInfo(nodes, v, newManager)); }
private DrInputStreamManager CreateInputNode(DryadLINQApp app, VertexInfo info, string inputName) { DrInputStreamManager s; int err = 0; DryadLogger.LogMethodEntry(inputName); if (info.ioType == VertexInfo.IOType.PARTITIONEDFILE) { DrPartitionInputStream input = new DrPartitionInputStream(); err = input.Open(app.GetUniverse(), info.sources[0]); if (!SUCCEEDED(err)) { string msg = String.Format("Could not read DSC input file {0}", info.sources[0]); throw new LinqToDryadException(msg, err); } DrManagerBase inputStage = new DrManagerBase(app.GetGraph(), inputName); DrInputStreamManager inputManager = new DrInputStreamManager(input, inputStage); s = inputManager; } //else if ( info.ioType == VertexInfo.IOType.STREAM ) //{ // DrDscInputStream input = new DrDscInputStream(); // DryadLogger.LogInformation("Create input node", "Opening DSC input fileset"); // err = input.Open(app.GetUniverse(), info.sources[0]); // if (!SUCCEEDED(err)) // { // string msg = String.Format("Could not read DSC input fileset {0}", info.sources[0]); // throw new LinqToDryadException(msg, err); // } // DryadLogger.LogInformation("Create input node", "Opened DSC input fileset"); // DrManagerBase inputStage = new DrManagerBase(app.GetGraph(), inputName); // DrInputStreamManager inputManager = new DrInputStreamManager(input, inputStage); // s = inputManager; //} else if (info.ioType == VertexInfo.IOType.HDFS_STREAM) { DrHdfsInputStream input = new DrHdfsInputStream(); DryadLogger.LogInformation("Create input node", "Opening HDFS input fileset"); err = input.Open(app.GetUniverse(), info.sources[0]); if (!SUCCEEDED(err)) { string msg = String.Format("Could not read HDFS input fileset {0}", info.sources[0]); throw new LinqToDryadException(msg, err); } DryadLogger.LogInformation("Create input node", "Opened HDFS input fileset"); DrManagerBase inputStage = new DrManagerBase(app.GetGraph(), inputName); DrInputStreamManager inputManager = new DrInputStreamManager(input, inputStage); s = inputManager; } else { string msg = String.Format("Unknown input type {0}", info.ioType); throw new LinqToDryadException(msg); } DryadLogger.LogMethodExit(); return(s); }