/// <exception cref="System.Exception"/>
        private byte[] GetFileContentsUsingDfs(string fileName, int len)
        {
            FSDataInputStream @in = hdfs.Open(new Path(fileName));

            byte[] ret = new byte[len];
            @in.ReadFully(ret);
            try
            {
                @in.ReadByte();
                NUnit.Framework.Assert.Fail("expected end of file");
            }
            catch (EOFException)
            {
            }
            // expected. Unfortunately there is no associated message to check
            @in.Close();
            return(ret);
        }
Beispiel #2
0
            /// <exception cref="System.IO.IOException"/>
            protected internal override InputStream GetInputStream(PathData item)
            {
                FSDataInputStream i = (FSDataInputStream)base.GetInputStream(item);
                // Handle 0 and 1-byte files
                short leadBytes;

                try
                {
                    leadBytes = i.ReadShort();
                }
                catch (EOFException)
                {
                    i.Seek(0);
                    return(i);
                }
                switch (leadBytes)
                {
                case unchecked ((int)(0x1f8b)):
                {
                    // Check type of stream first
                    // RFC 1952
                    // Must be gzip
                    i.Seek(0);
                    return(new GZIPInputStream(i));
                }

                case unchecked ((int)(0x5345)):
                {
                    // 'S' 'E'
                    // Might be a SequenceFile
                    if (i.ReadByte() == 'Q')
                    {
                        i.Close();
                        return(new Display.TextRecordInputStream(this, item.stat));
                    }
                    goto default;
                }

                default:
                {
                    // Check the type of compression instead, depending on Codec class's
                    // own detection methods, based on the provided path.
                    CompressionCodecFactory cf    = new CompressionCodecFactory(GetConf());
                    CompressionCodec        codec = cf.GetCodec(item.path);
                    if (codec != null)
                    {
                        i.Seek(0);
                        return(codec.CreateInputStream(i));
                    }
                    break;
                }

                case unchecked ((int)(0x4f62)):
                {
                    // 'O' 'b'
                    if (i.ReadByte() == 'j')
                    {
                        i.Close();
                        return(new Display.AvroFileInputStream(item.stat));
                    }
                    break;
                }
                }
                // File is non-compressed, or not a file container we know.
                i.Seek(0);
                return(i);
            }