private DrInputStreamManager CreateInputNode(DryadLINQApp app, VertexInfo info, string inputName) { DrInputStreamManager s; int err = 0; DryadLogger.LogMethodEntry(inputName); if (info.ioType == VertexInfo.IOType.PARTITIONEDFILE) { DrPartitionInputStream input = new DrPartitionInputStream(); err = input.Open(app.GetUniverse(), info.sources[0]); if (!SUCCEEDED(err)) { string msg = String.Format("Could not read DSC input file {0}", info.sources[0]); throw new LinqToDryadException(msg, err); } DrManagerBase inputStage = new DrManagerBase(app.GetGraph(), inputName); DrInputStreamManager inputManager = new DrInputStreamManager(input, inputStage); s = inputManager; } //else if ( info.ioType == VertexInfo.IOType.STREAM ) //{ // DrDscInputStream input = new DrDscInputStream(); // DryadLogger.LogInformation("Create input node", "Opening DSC input fileset"); // err = input.Open(app.GetUniverse(), info.sources[0]); // if (!SUCCEEDED(err)) // { // string msg = String.Format("Could not read DSC input fileset {0}", info.sources[0]); // throw new LinqToDryadException(msg, err); // } // DryadLogger.LogInformation("Create input node", "Opened DSC input fileset"); // DrManagerBase inputStage = new DrManagerBase(app.GetGraph(), inputName); // DrInputStreamManager inputManager = new DrInputStreamManager(input, inputStage); // s = inputManager; //} else if (info.ioType == VertexInfo.IOType.HDFS_STREAM) { DrHdfsInputStream input = new DrHdfsInputStream(); DryadLogger.LogInformation("Create input node", "Opening HDFS input fileset"); err = input.Open(app.GetUniverse(), info.sources[0]); if (!SUCCEEDED(err)) { string msg = String.Format("Could not read HDFS input fileset {0}", info.sources[0]); throw new LinqToDryadException(msg, err); } DryadLogger.LogInformation("Create input node", "Opened HDFS input fileset"); DrManagerBase inputStage = new DrManagerBase(app.GetGraph(), inputName); DrInputStreamManager inputManager = new DrInputStreamManager(input, inputStage); s = inputManager; } else { string msg = String.Format("Unknown input type {0}", info.ioType); throw new LinqToDryadException(msg); } DryadLogger.LogMethodExit(); return(s); }
private DrOutputStreamManager CreateOutputNode(DryadLINQApp app, VertexInfo info, string outputName) { DrOutputStreamManager s; if (info.ioType == VertexInfo.IOType.PARTITIONEDFILE) { DrPartitionOutputStream output = new DrPartitionOutputStream(); int err = output.Open(info.sources[0], info.partitionUncPath); if (!SUCCEEDED(err)) { string msg = String.Format("Could not open DSC output fileset {0}", info.sources[0]); throw new LinqToDryadException(msg, err); } DrManagerBase outputStage = new DrManagerBase(app.GetGraph(), outputName); DrOutputStreamManager outputManager = new DrOutputStreamManager(output, outputStage); app.GetGraph().AddPartitionGenerator(outputManager); s = outputManager; } //else if ( info.ioType == VertexInfo.IOType.STREAM ) //{ // DrDscOutputStream output = new DrDscOutputStream(info.compressionScheme, info.isTemporary); // int err = 0; // if (info.recordType == "") // { // err = output.Open(info.sources[0], info.partitionUncPath); // } // else // { // err = output.OpenWithRecordType(info.sources[0], info.partitionUncPath, info.recordType); // } // if (!SUCCEEDED(err)) // { // string msg = String.Format("Could not open DSC output fileset {0}", info.sources[0]); // throw new LinqToDryadException(msg, err); // } // DrManagerBase outputStage = new DrManagerBase(app.GetGraph(), outputName); // DrOutputStreamManager outputManager = new DrOutputStreamManager(output, outputStage); // app.GetGraph().AddPartitionGenerator(outputManager); // s = outputManager; //} else if (info.ioType == VertexInfo.IOType.HDFS_STREAM) { DrHdfsOutputStream output = new DrHdfsOutputStream(); int err = output.Open(info.sources[0]); if (!SUCCEEDED(err)) { string msg = String.Format("Could not open HDFS output fileset {0}", info.sources[0]); throw new LinqToDryadException(msg, err); } DrManagerBase outputStage = new DrManagerBase(app.GetGraph(), outputName); DrOutputStreamManager outputManager = new DrOutputStreamManager(output, outputStage); app.GetGraph().AddPartitionGenerator(outputManager); s = outputManager; } else { string msg = String.Format("Unknown output type {0}", info.ioType); throw new LinqToDryadException(msg); } return(s); }