Example #1
0
        private DrInputStreamManager CreateInputNode(DryadLINQApp app, VertexInfo info, string inputName)
        {
            DrInputStreamManager s;
            int err = 0;

            DryadLogger.LogInformation("methodEntry: " + inputName);

            if (info.ioType == VertexInfo.IOType.PARTITIONEDFILE)
            {
                DrPartitionInputStream input = new DrPartitionInputStream();

                err = input.Open(app.GetUniverse(), new Uri(info.sources[0]).AbsolutePath);
                if (!SUCCEEDED(err))
                {
                    string msg = String.Format("Could not read Partitioned file input {0}", info.sources[0]);
                    throw new LinqToDryadException(msg, err);
                }

                DrManagerBase        inputStage   = new DrManagerBase(app.GetGraph(), inputName);
                DrInputStreamManager inputManager = new DrInputStreamManager(input, inputStage);

                s = inputManager;
            }
            //else if ( info.ioType == VertexInfo.IOType.STREAM )
            //{
            //    DrDscInputStream input = new DrDscInputStream();

            //    DryadLogger.LogInformation("Create input node", "Opening DSC input fileset");

            //    err = input.Open(app.GetUniverse(), info.sources[0]);
            //    if (!SUCCEEDED(err))
            //    {
            //        string msg = String.Format("Could not read DSC input fileset {0}", info.sources[0]);
            //        throw new LinqToDryadException(msg, err);
            //    }

            //    DryadLogger.LogInformation("Create input node", "Opened DSC input fileset");

            //    DrManagerBase inputStage = new DrManagerBase(app.GetGraph(), inputName);
            //    DrInputStreamManager inputManager = new DrInputStreamManager(input, inputStage);

            //    s = inputManager;
            //}
            else if (info.ioType == VertexInfo.IOType.HDFS_STREAM)
            {
                DrHdfsInputStream input = new DrHdfsInputStream();

                DryadLogger.LogInformation("Create input node", "Opening HDFS input fileset");

                Uri srcUri = new Uri(info.sources[0]);

                err = input.Open(app.GetUniverse(), srcUri.GetLeftPart(UriPartial.Path), info.recordType);
                if (!SUCCEEDED(err))
                {
                    string msg = String.Format("Could not read HDFS input fileset {0}: {1}", info.sources[0], input.GetError());
                    throw new LinqToDryadException(msg, err);
                }

                DryadLogger.LogInformation("Create input node", "Opened HDFS input fileset");

                DrManagerBase        inputStage   = new DrManagerBase(app.GetGraph(), inputName);
                DrInputStreamManager inputManager = new DrInputStreamManager(input, inputStage);

                s = inputManager;
            }
            else if (info.ioType == VertexInfo.IOType.AZUREBLOB)
            {
                DrAzureInputStream input = new DrAzureInputStream();

                DryadLogger.LogInformation("Create input node", "Opening Azure input fileset");

                try
                {
                    input.Open(info.sources[0]);
                }
                catch (Exception e)
                {
                    string msg = String.Format("Could not read Azure input fileset {0}: {1}", info.sources[0], e.ToString());
                    throw new LinqToDryadException(msg);
                }

                DryadLogger.LogInformation("Create input node", "Opened Azure input fileset");

                DrManagerBase        inputStage   = new DrManagerBase(app.GetGraph(), inputName);
                DrInputStreamManager inputManager = new DrInputStreamManager(input, inputStage);

                s = inputManager;
            }
            else
            {
                string msg = String.Format("Unknown input type {0}", info.ioType);
                throw new LinqToDryadException(msg);
            }

            DryadLogger.LogInformation("methodExit");
            return(s);
        }
Example #2
0
        private DrOutputStreamManager CreateOutputNode(DryadLINQApp app, VertexInfo info, string outputName)
        {
            DryadLogger.LogInformation("methodEntry: " + outputName);

            DrOutputStreamManager s;

            if (info.ioType == VertexInfo.IOType.PARTITIONEDFILE)
            {
                DrPartitionOutputStream output = new DrPartitionOutputStream();
                Uri    sourceUri  = new Uri(info.sources[0]);
                string sourcePath = sourceUri.AbsolutePath;
                if (!String.IsNullOrEmpty(sourceUri.Host))
                {
                    sourcePath = @"\\" + sourceUri.Host + sourcePath;
                }
                int err = output.Open(sourcePath, info.partitionUncPath);
                if (!SUCCEEDED(err))
                {
                    string msg = String.Format("Could not open output fileset {0}", sourcePath);
                    throw new LinqToDryadException(msg, err);
                }

                DrManagerBase         outputStage   = new DrManagerBase(app.GetGraph(), outputName);
                DrOutputStreamManager outputManager = new DrOutputStreamManager(output, outputStage);
                app.GetGraph().AddPartitionGenerator(outputManager);

                s = outputManager;
            }
            //else if ( info.ioType == VertexInfo.IOType.STREAM )
            //{
            //    DrDscOutputStream output = new DrDscOutputStream(info.compressionScheme, info.isTemporary);
            //    int err = 0;
            //    if (info.recordType == "")
            //    {
            //        err = output.Open(info.sources[0], info.partitionUncPath);
            //    }
            //    else
            //    {
            //        err = output.OpenWithRecordType(info.sources[0], info.partitionUncPath, info.recordType);
            //    }

            //    if (!SUCCEEDED(err))
            //    {
            //        string msg = String.Format("Could not open DSC output fileset {0}", info.sources[0]);
            //        throw new LinqToDryadException(msg, err);
            //    }

            //    DrManagerBase outputStage = new DrManagerBase(app.GetGraph(), outputName);
            //    DrOutputStreamManager outputManager = new DrOutputStreamManager(output, outputStage);
            //    app.GetGraph().AddPartitionGenerator(outputManager);

            //    s = outputManager;
            //}
            else if (info.ioType == VertexInfo.IOType.HDFS_STREAM)
            {
                DrHdfsOutputStream output = new DrHdfsOutputStream();
                int err = output.Open(info.sources[0]);

                if (!SUCCEEDED(err))
                {
                    string msg = String.Format("Could not open HDFS output fileset {0}: {1}", info.sources[0], output.GetError());
                    throw new LinqToDryadException(msg, err);
                }

                DrManagerBase         outputStage   = new DrManagerBase(app.GetGraph(), outputName);
                DrOutputStreamManager outputManager = new DrOutputStreamManager(output, outputStage);
                app.GetGraph().AddPartitionGenerator(outputManager);

                s = outputManager;
            }
            else if (info.ioType == VertexInfo.IOType.AZUREBLOB)
            {
                DrAzureOutputStream output = new DrAzureOutputStream();

                try
                {
                    output.Open(info.sources[0]);
                }
                catch (Exception e)
                {
                    string msg = String.Format("Could not open Azure output fileset {0}: {1}", info.sources[0], e.ToString());
                    throw new LinqToDryadException(msg);
                }

                DrManagerBase         outputStage   = new DrManagerBase(app.GetGraph(), outputName);
                DrOutputStreamManager outputManager = new DrOutputStreamManager(output, outputStage);
                app.GetGraph().AddPartitionGenerator(outputManager);

                s = outputManager;
            }
            else
            {
                string msg = String.Format("Unknown output type {0}", info.ioType);
                throw new LinqToDryadException(msg);
            }

            return(s);
        }