private DrInputStreamManager CreateInputNode(DryadLINQApp app, VertexInfo info, string inputName)
{
DrInputStreamManager s;
int err = 0;
DryadLogger.LogMethodEntry(inputName);
if (info.ioType == VertexInfo.IOType.PARTITIONEDFILE)
{
DrPartitionInputStream input = new DrPartitionInputStream();
err = input.Open(app.GetUniverse(), info.sources[0]);
if (!SUCCEEDED(err))
{
string msg = String.Format("Could not read DSC input file {0}", info.sources[0]);
throw new LinqToDryadException(msg, err);
}
DrManagerBase inputStage = new DrManagerBase(app.GetGraph(), inputName);
DrInputStreamManager inputManager = new DrInputStreamManager(input, inputStage);
s = inputManager;
}
//else if ( info.ioType == VertexInfo.IOType.STREAM )
//{
// DrDscInputStream input = new DrDscInputStream();
// DryadLogger.LogInformation("Create input node", "Opening DSC input fileset");
// err = input.Open(app.GetUniverse(), info.sources[0]);
// if (!SUCCEEDED(err))
// {
// string msg = String.Format("Could not read DSC input fileset {0}", info.sources[0]);
// throw new LinqToDryadException(msg, err);
// }
// DryadLogger.LogInformation("Create input node", "Opened DSC input fileset");
// DrManagerBase inputStage = new DrManagerBase(app.GetGraph(), inputName);
// DrInputStreamManager inputManager = new DrInputStreamManager(input, inputStage);
// s = inputManager;
//}
else if (info.ioType == VertexInfo.IOType.HDFS_STREAM)
{
DrHdfsInputStream input = new DrHdfsInputStream();
DryadLogger.LogInformation("Create input node", "Opening HDFS input fileset");
err = input.Open(app.GetUniverse(), info.sources[0]);
if (!SUCCEEDED(err))
{
string msg = String.Format("Could not read HDFS input fileset {0}", info.sources[0]);
throw new LinqToDryadException(msg, err);
}
DryadLogger.LogInformation("Create input node", "Opened HDFS input fileset");
DrManagerBase inputStage = new DrManagerBase(app.GetGraph(), inputName);
DrInputStreamManager inputManager = new DrInputStreamManager(input, inputStage);
s = inputManager;
}
else
{
string msg = String.Format("Unknown input type {0}", info.ioType);
throw new LinqToDryadException(msg);
}
DryadLogger.LogMethodExit();
return(s);
}
private DrOutputStreamManager CreateOutputNode(DryadLINQApp app, VertexInfo info, string outputName)
{
DrOutputStreamManager s;
if (info.ioType == VertexInfo.IOType.PARTITIONEDFILE)
{
DrPartitionOutputStream output = new DrPartitionOutputStream();
int err = output.Open(info.sources[0], info.partitionUncPath);
if (!SUCCEEDED(err))
{
string msg = String.Format("Could not open DSC output fileset {0}", info.sources[0]);
throw new LinqToDryadException(msg, err);
}
DrManagerBase outputStage = new DrManagerBase(app.GetGraph(), outputName);
DrOutputStreamManager outputManager = new DrOutputStreamManager(output, outputStage);
app.GetGraph().AddPartitionGenerator(outputManager);
s = outputManager;
}
//else if ( info.ioType == VertexInfo.IOType.STREAM )
//{
// DrDscOutputStream output = new DrDscOutputStream(info.compressionScheme, info.isTemporary);
// int err = 0;
// if (info.recordType == "")
// {
// err = output.Open(info.sources[0], info.partitionUncPath);
// }
// else
// {
// err = output.OpenWithRecordType(info.sources[0], info.partitionUncPath, info.recordType);
// }
// if (!SUCCEEDED(err))
// {
// string msg = String.Format("Could not open DSC output fileset {0}", info.sources[0]);
// throw new LinqToDryadException(msg, err);
// }
// DrManagerBase outputStage = new DrManagerBase(app.GetGraph(), outputName);
// DrOutputStreamManager outputManager = new DrOutputStreamManager(output, outputStage);
// app.GetGraph().AddPartitionGenerator(outputManager);
// s = outputManager;
//}
else if (info.ioType == VertexInfo.IOType.HDFS_STREAM)
{
DrHdfsOutputStream output = new DrHdfsOutputStream();
int err = output.Open(info.sources[0]);
if (!SUCCEEDED(err))
{
string msg = String.Format("Could not open HDFS output fileset {0}", info.sources[0]);
throw new LinqToDryadException(msg, err);
}
DrManagerBase outputStage = new DrManagerBase(app.GetGraph(), outputName);
DrOutputStreamManager outputManager = new DrOutputStreamManager(output, outputStage);
app.GetGraph().AddPartitionGenerator(outputManager);
s = outputManager;
}
else
{
string msg = String.Format("Unknown output type {0}", info.ioType);
throw new LinqToDryadException(msg);
}
return(s);
}
