private DrOutputStreamManager CreateOutputNode(DryadLINQApp app, VertexInfo info, string outputName)
{
DryadLogger.LogInformation("methodEntry: " + outputName);
DrOutputStreamManager s;
if (info.ioType == VertexInfo.IOType.PARTITIONEDFILE)
{
DrPartitionOutputStream output = new DrPartitionOutputStream();
Uri sourceUri = new Uri(info.sources[0]);
string sourcePath = sourceUri.AbsolutePath;
if (!String.IsNullOrEmpty(sourceUri.Host))
{
sourcePath = @"\\" + sourceUri.Host + sourcePath;
}
int err = output.Open(sourcePath, info.partitionUncPath);
if (!SUCCEEDED(err))
{
string msg = String.Format("Could not open output fileset {0}", sourcePath);
throw new LinqToDryadException(msg, err);
}
DrManagerBase outputStage = new DrManagerBase(app.GetGraph(), outputName);
DrOutputStreamManager outputManager = new DrOutputStreamManager(output, outputStage);
app.GetGraph().AddPartitionGenerator(outputManager);
s = outputManager;
}
//else if ( info.ioType == VertexInfo.IOType.STREAM )
//{
// DrDscOutputStream output = new DrDscOutputStream(info.compressionScheme, info.isTemporary);
// int err = 0;
// if (info.recordType == "")
// {
// err = output.Open(info.sources[0], info.partitionUncPath);
// }
// else
// {
// err = output.OpenWithRecordType(info.sources[0], info.partitionUncPath, info.recordType);
// }
// if (!SUCCEEDED(err))
// {
// string msg = String.Format("Could not open DSC output fileset {0}", info.sources[0]);
// throw new LinqToDryadException(msg, err);
// }
// DrManagerBase outputStage = new DrManagerBase(app.GetGraph(), outputName);
// DrOutputStreamManager outputManager = new DrOutputStreamManager(output, outputStage);
// app.GetGraph().AddPartitionGenerator(outputManager);
// s = outputManager;
//}
else if (info.ioType == VertexInfo.IOType.HDFS_STREAM)
{
DrHdfsOutputStream output = new DrHdfsOutputStream();
int err = output.Open(info.sources[0]);
if (!SUCCEEDED(err))
{
string msg = String.Format("Could not open HDFS output fileset {0}: {1}", info.sources[0], output.GetError());
throw new LinqToDryadException(msg, err);
}
DrManagerBase outputStage = new DrManagerBase(app.GetGraph(), outputName);
DrOutputStreamManager outputManager = new DrOutputStreamManager(output, outputStage);
app.GetGraph().AddPartitionGenerator(outputManager);
s = outputManager;
}
else if (info.ioType == VertexInfo.IOType.AZUREBLOB)
{
DrAzureOutputStream output = new DrAzureOutputStream();
try
{
output.Open(info.sources[0]);
}
catch (Exception e)
{
string msg = String.Format("Could not open Azure output fileset {0}: {1}", info.sources[0], e.ToString());
throw new LinqToDryadException(msg);
}
DrManagerBase outputStage = new DrManagerBase(app.GetGraph(), outputName);
DrOutputStreamManager outputManager = new DrOutputStreamManager(output, outputStage);
app.GetGraph().AddPartitionGenerator(outputManager);
s = outputManager;
}
else
{
string msg = String.Format("Unknown output type {0}", info.ioType);
throw new LinqToDryadException(msg);
}
return(s);
}
private DrOutputStreamManager CreateOutputNode(DryadLINQApp app, VertexInfo info, string outputName)
{
DrOutputStreamManager s;
if (info.ioType == VertexInfo.IOType.PARTITIONEDFILE)
{
DrPartitionOutputStream output = new DrPartitionOutputStream();
int err = output.Open(info.sources[0], info.partitionUncPath);
if (!SUCCEEDED(err))
{
string msg = String.Format("Could not open DSC output fileset {0}", info.sources[0]);
throw new LinqToDryadException(msg, err);
}
DrManagerBase outputStage = new DrManagerBase(app.GetGraph(), outputName);
DrOutputStreamManager outputManager = new DrOutputStreamManager(output, outputStage);
app.GetGraph().AddPartitionGenerator(outputManager);
s = outputManager;
}
//else if ( info.ioType == VertexInfo.IOType.STREAM )
//{
// DrDscOutputStream output = new DrDscOutputStream(info.compressionScheme, info.isTemporary);
// int err = 0;
// if (info.recordType == "")
// {
// err = output.Open(info.sources[0], info.partitionUncPath);
// }
// else
// {
// err = output.OpenWithRecordType(info.sources[0], info.partitionUncPath, info.recordType);
// }
// if (!SUCCEEDED(err))
// {
// string msg = String.Format("Could not open DSC output fileset {0}", info.sources[0]);
// throw new LinqToDryadException(msg, err);
// }
// DrManagerBase outputStage = new DrManagerBase(app.GetGraph(), outputName);
// DrOutputStreamManager outputManager = new DrOutputStreamManager(output, outputStage);
// app.GetGraph().AddPartitionGenerator(outputManager);
// s = outputManager;
//}
else if (info.ioType == VertexInfo.IOType.HDFS_STREAM)
{
DrHdfsOutputStream output = new DrHdfsOutputStream();
int err = output.Open(info.sources[0]);
if (!SUCCEEDED(err))
{
string msg = String.Format("Could not open HDFS output fileset {0}", info.sources[0]);
throw new LinqToDryadException(msg, err);
}
DrManagerBase outputStage = new DrManagerBase(app.GetGraph(), outputName);
DrOutputStreamManager outputManager = new DrOutputStreamManager(output, outputStage);
app.GetGraph().AddPartitionGenerator(outputManager);
s = outputManager;
}
else
{
string msg = String.Format("Unknown output type {0}", info.ioType);
throw new LinqToDryadException(msg);
}
return(s);
}
