private DrInputStreamManager CreateInputNode(DryadLINQApp app, VertexInfo info, string inputName)
{
DrInputStreamManager s;
int err = 0;
DryadLogger.LogInformation("methodEntry: " + inputName);
if (info.ioType == VertexInfo.IOType.PARTITIONEDFILE)
{
DrPartitionInputStream input = new DrPartitionInputStream();
err = input.Open(app.GetUniverse(), new Uri(info.sources[0]).AbsolutePath);
if (!SUCCEEDED(err))
{
string msg = String.Format("Could not read Partitioned file input {0}", info.sources[0]);
throw new LinqToDryadException(msg, err);
}
DrManagerBase inputStage = new DrManagerBase(app.GetGraph(), inputName);
DrInputStreamManager inputManager = new DrInputStreamManager(input, inputStage);
s = inputManager;
}
//else if ( info.ioType == VertexInfo.IOType.STREAM )
//{
// DrDscInputStream input = new DrDscInputStream();
// DryadLogger.LogInformation("Create input node", "Opening DSC input fileset");
// err = input.Open(app.GetUniverse(), info.sources[0]);
// if (!SUCCEEDED(err))
// {
// string msg = String.Format("Could not read DSC input fileset {0}", info.sources[0]);
// throw new LinqToDryadException(msg, err);
// }
// DryadLogger.LogInformation("Create input node", "Opened DSC input fileset");
// DrManagerBase inputStage = new DrManagerBase(app.GetGraph(), inputName);
// DrInputStreamManager inputManager = new DrInputStreamManager(input, inputStage);
// s = inputManager;
//}
else if (info.ioType == VertexInfo.IOType.HDFS_STREAM)
{
DrHdfsInputStream input = new DrHdfsInputStream();
DryadLogger.LogInformation("Create input node", "Opening HDFS input fileset");
Uri srcUri = new Uri(info.sources[0]);
err = input.Open(app.GetUniverse(), srcUri.GetLeftPart(UriPartial.Path), info.recordType);
if (!SUCCEEDED(err))
{
string msg = String.Format("Could not read HDFS input fileset {0}: {1}", info.sources[0], input.GetError());
throw new LinqToDryadException(msg, err);
}
DryadLogger.LogInformation("Create input node", "Opened HDFS input fileset");
DrManagerBase inputStage = new DrManagerBase(app.GetGraph(), inputName);
DrInputStreamManager inputManager = new DrInputStreamManager(input, inputStage);
s = inputManager;
}
else if (info.ioType == VertexInfo.IOType.AZUREBLOB)
{
DrAzureInputStream input = new DrAzureInputStream();
DryadLogger.LogInformation("Create input node", "Opening Azure input fileset");
try
{
input.Open(info.sources[0]);
}
catch (Exception e)
{
string msg = String.Format("Could not read Azure input fileset {0}: {1}", info.sources[0], e.ToString());
throw new LinqToDryadException(msg);
}
DryadLogger.LogInformation("Create input node", "Opened Azure input fileset");
DrManagerBase inputStage = new DrManagerBase(app.GetGraph(), inputName);
DrInputStreamManager inputManager = new DrInputStreamManager(input, inputStage);
s = inputManager;
}
else
{
string msg = String.Format("Unknown input type {0}", info.ioType);
throw new LinqToDryadException(msg);
}
DryadLogger.LogInformation("methodExit");
return(s);
}
private DrInputStreamManager CreateInputNode(DryadLINQApp app, VertexInfo info, string inputName)
{
DrInputStreamManager s;
int err = 0;
DryadLogger.LogMethodEntry(inputName);
if (info.ioType == VertexInfo.IOType.PARTITIONEDFILE)
{
DrPartitionInputStream input = new DrPartitionInputStream();
err = input.Open(app.GetUniverse(), info.sources[0]);
if (!SUCCEEDED(err))
{
string msg = String.Format("Could not read DSC input file {0}", info.sources[0]);
throw new LinqToDryadException(msg, err);
}
DrManagerBase inputStage = new DrManagerBase(app.GetGraph(), inputName);
DrInputStreamManager inputManager = new DrInputStreamManager(input, inputStage);
s = inputManager;
}
//else if ( info.ioType == VertexInfo.IOType.STREAM )
//{
// DrDscInputStream input = new DrDscInputStream();
// DryadLogger.LogInformation("Create input node", "Opening DSC input fileset");
// err = input.Open(app.GetUniverse(), info.sources[0]);
// if (!SUCCEEDED(err))
// {
// string msg = String.Format("Could not read DSC input fileset {0}", info.sources[0]);
// throw new LinqToDryadException(msg, err);
// }
// DryadLogger.LogInformation("Create input node", "Opened DSC input fileset");
// DrManagerBase inputStage = new DrManagerBase(app.GetGraph(), inputName);
// DrInputStreamManager inputManager = new DrInputStreamManager(input, inputStage);
// s = inputManager;
//}
else if (info.ioType == VertexInfo.IOType.HDFS_STREAM)
{
DrHdfsInputStream input = new DrHdfsInputStream();
DryadLogger.LogInformation("Create input node", "Opening HDFS input fileset");
err = input.Open(app.GetUniverse(), info.sources[0]);
if (!SUCCEEDED(err))
{
string msg = String.Format("Could not read HDFS input fileset {0}", info.sources[0]);
throw new LinqToDryadException(msg, err);
}
DryadLogger.LogInformation("Create input node", "Opened HDFS input fileset");
DrManagerBase inputStage = new DrManagerBase(app.GetGraph(), inputName);
DrInputStreamManager inputManager = new DrInputStreamManager(input, inputStage);
s = inputManager;
}
else
{
string msg = String.Format("Unknown input type {0}", info.ioType);
throw new LinqToDryadException(msg);
}
DryadLogger.LogMethodExit();
return(s);
}
