private DrInputStreamManager CreateInputNode(DryadLINQApp app, VertexInfo info, string inputName)
{
DrInputStreamManager s;
int err = 0;
DryadLogger.LogInformation("methodEntry: " + inputName);
if (info.ioType == VertexInfo.IOType.PARTITIONEDFILE)
{
DrPartitionInputStream input = new DrPartitionInputStream();
err = input.Open(app.GetUniverse(), new Uri(info.sources[0]).AbsolutePath);
if (!SUCCEEDED(err))
{
string msg = String.Format("Could not read Partitioned file input {0}", info.sources[0]);
throw new LinqToDryadException(msg, err);
}
DrManagerBase inputStage = new DrManagerBase(app.GetGraph(), inputName);
DrInputStreamManager inputManager = new DrInputStreamManager(input, inputStage);
s = inputManager;
}
//else if ( info.ioType == VertexInfo.IOType.STREAM )
//{
// DrDscInputStream input = new DrDscInputStream();
// DryadLogger.LogInformation("Create input node", "Opening DSC input fileset");
// err = input.Open(app.GetUniverse(), info.sources[0]);
// if (!SUCCEEDED(err))
// {
// string msg = String.Format("Could not read DSC input fileset {0}", info.sources[0]);
// throw new LinqToDryadException(msg, err);
// }
// DryadLogger.LogInformation("Create input node", "Opened DSC input fileset");
// DrManagerBase inputStage = new DrManagerBase(app.GetGraph(), inputName);
// DrInputStreamManager inputManager = new DrInputStreamManager(input, inputStage);
// s = inputManager;
//}
else if (info.ioType == VertexInfo.IOType.HDFS_STREAM)
{
DrHdfsInputStream input = new DrHdfsInputStream();
DryadLogger.LogInformation("Create input node", "Opening HDFS input fileset");
Uri srcUri = new Uri(info.sources[0]);
err = input.Open(app.GetUniverse(), srcUri.GetLeftPart(UriPartial.Path), info.recordType);
if (!SUCCEEDED(err))
{
string msg = String.Format("Could not read HDFS input fileset {0}: {1}", info.sources[0], input.GetError());
throw new LinqToDryadException(msg, err);
}
DryadLogger.LogInformation("Create input node", "Opened HDFS input fileset");
DrManagerBase inputStage = new DrManagerBase(app.GetGraph(), inputName);
DrInputStreamManager inputManager = new DrInputStreamManager(input, inputStage);
s = inputManager;
}
else if (info.ioType == VertexInfo.IOType.AZUREBLOB)
{
DrAzureInputStream input = new DrAzureInputStream();
DryadLogger.LogInformation("Create input node", "Opening Azure input fileset");
try
{
input.Open(info.sources[0]);
}
catch (Exception e)
{
string msg = String.Format("Could not read Azure input fileset {0}: {1}", info.sources[0], e.ToString());
throw new LinqToDryadException(msg);
}
DryadLogger.LogInformation("Create input node", "Opened Azure input fileset");
DrManagerBase inputStage = new DrManagerBase(app.GetGraph(), inputName);
DrInputStreamManager inputManager = new DrInputStreamManager(input, inputStage);
s = inputManager;
}
else
{
string msg = String.Format("Unknown input type {0}", info.ioType);
throw new LinqToDryadException(msg);
}
DryadLogger.LogInformation("methodExit");
return(s);
}
private DrOutputStreamManager CreateOutputNode(DryadLINQApp app, VertexInfo info, string outputName)
{
DryadLogger.LogInformation("methodEntry: " + outputName);
DrOutputStreamManager s;
if (info.ioType == VertexInfo.IOType.PARTITIONEDFILE)
{
DrPartitionOutputStream output = new DrPartitionOutputStream();
Uri sourceUri = new Uri(info.sources[0]);
string sourcePath = sourceUri.AbsolutePath;
if (!String.IsNullOrEmpty(sourceUri.Host))
{
sourcePath = @"\\" + sourceUri.Host + sourcePath;
}
int err = output.Open(sourcePath, info.partitionUncPath);
if (!SUCCEEDED(err))
{
string msg = String.Format("Could not open output fileset {0}", sourcePath);
throw new LinqToDryadException(msg, err);
}
DrManagerBase outputStage = new DrManagerBase(app.GetGraph(), outputName);
DrOutputStreamManager outputManager = new DrOutputStreamManager(output, outputStage);
app.GetGraph().AddPartitionGenerator(outputManager);
s = outputManager;
}
//else if ( info.ioType == VertexInfo.IOType.STREAM )
//{
// DrDscOutputStream output = new DrDscOutputStream(info.compressionScheme, info.isTemporary);
// int err = 0;
// if (info.recordType == "")
// {
// err = output.Open(info.sources[0], info.partitionUncPath);
// }
// else
// {
// err = output.OpenWithRecordType(info.sources[0], info.partitionUncPath, info.recordType);
// }
// if (!SUCCEEDED(err))
// {
// string msg = String.Format("Could not open DSC output fileset {0}", info.sources[0]);
// throw new LinqToDryadException(msg, err);
// }
// DrManagerBase outputStage = new DrManagerBase(app.GetGraph(), outputName);
// DrOutputStreamManager outputManager = new DrOutputStreamManager(output, outputStage);
// app.GetGraph().AddPartitionGenerator(outputManager);
// s = outputManager;
//}
else if (info.ioType == VertexInfo.IOType.HDFS_STREAM)
{
DrHdfsOutputStream output = new DrHdfsOutputStream();
int err = output.Open(info.sources[0]);
if (!SUCCEEDED(err))
{
string msg = String.Format("Could not open HDFS output fileset {0}: {1}", info.sources[0], output.GetError());
throw new LinqToDryadException(msg, err);
}
DrManagerBase outputStage = new DrManagerBase(app.GetGraph(), outputName);
DrOutputStreamManager outputManager = new DrOutputStreamManager(output, outputStage);
app.GetGraph().AddPartitionGenerator(outputManager);
s = outputManager;
}
else if (info.ioType == VertexInfo.IOType.AZUREBLOB)
{
DrAzureOutputStream output = new DrAzureOutputStream();
try
{
output.Open(info.sources[0]);
}
catch (Exception e)
{
string msg = String.Format("Could not open Azure output fileset {0}: {1}", info.sources[0], e.ToString());
throw new LinqToDryadException(msg);
}
DrManagerBase outputStage = new DrManagerBase(app.GetGraph(), outputName);
DrOutputStreamManager outputManager = new DrOutputStreamManager(output, outputStage);
app.GetGraph().AddPartitionGenerator(outputManager);
s = outputManager;
}
else
{
string msg = String.Format("Unknown output type {0}", info.ioType);
throw new LinqToDryadException(msg);
}
return(s);
}