internal DryadLinqBinaryReader(NativeBlockStream stream, Encoding encoding)
 {
     this.m_nativeStream = stream;
     this.m_encoding = encoding;
     this.m_decoder = encoding.GetDecoder();
     this.m_curDataBlockInfo.DataBlock = null;
     this.m_curDataBlockInfo.BlockSize = -1;
     this.m_curDataBlockInfo.ItemHandle = IntPtr.Zero;
     this.m_curDataBlock = this.m_curDataBlockInfo.DataBlock;
     this.m_curBlockSize = this.m_curDataBlockInfo.BlockSize;
     this.m_curBlockPos = -1;
     this.m_isClosed = false;
 }
Пример #2
0
 /// <summary>
 /// Initializes an instance of DryadLinqTextReader.
 /// </summary>
 /// <param name="stream">A native stream to read from.</param>
 /// <param name="encoding">The text encoding.</param>
 public DryadLinqTextReader(NativeBlockStream stream, Encoding encoding)
 {
     this.m_nativeStream = stream;
     this.m_encoding = encoding;
     this.m_decoder = encoding.GetDecoder();
     this.m_curDataBlockInfo.DataBlock = null;
     this.m_curDataBlockInfo.BlockSize = -1;
     this.m_curDataBlockInfo.ItemHandle = IntPtr.Zero;
     this.m_curBlockPos = 0;
     this.m_decodeUnitCharSize = this.m_encoding.GetMaxCharCount(DecodeUnitByteSize);
     this.m_charBuff = new char[this.m_decodeUnitCharSize + 2];  //allow 2 bytes for trailing newline
     this.m_charBuffEnd = 0;
     this.m_curLineStart = 0;
     this.m_curLineEnd = 0;
     this.m_isClosed = false;
 }
Пример #3
0
 internal MultiBlockStream(List<string[]> srcList,
                           string associatedDscStreamName,
                           FileAccess access,
                           CompressionScheme scheme)
 {
     this.m_srcList = srcList;
     this.m_associatedDscStreamName = associatedDscStreamName;
     if (srcList.Count == 0)
     {
         throw new DryadLinqException(DryadLinqErrorCode.MultiBlockEmptyPartitionList,
                                      SR.MultiBlockEmptyPartitionList);
     }
     this.m_compressionScheme = scheme;
     this.m_curIdx = 0;
     this.m_curStream = this.GetStream(this.m_curIdx++, access);
 }
Пример #4
0
 /// <summary>
 /// Initializes an instance of DryadLinqTextWriter.
 /// </summary>
 /// <param name="stream">A native stream to write to.</param>
 /// <param name="encoding">The text encoding.</param>
 /// <param name="buffSize">A hint for the size of write buffer.</param>
 public DryadLinqTextWriter(NativeBlockStream stream, Encoding encoding, Int32 buffSize)
 {
     this.m_nativeStream = stream;
     this.m_encoding = encoding;
     this.m_nextBlockSize = Math.Max(DefaultBlockSize, buffSize/2);
     this.m_bufferSizeHint = buffSize;
     this.m_curDataBlockInfo.DataBlock = null;
     this.m_curDataBlockInfo.BlockSize = 0;
     this.m_curDataBlockInfo.ItemHandle = IntPtr.Zero;            
     this.m_curDataBlock = this.m_curDataBlockInfo.DataBlock;
     this.m_curBlockSize = this.m_curDataBlockInfo.BlockSize;
     this.m_curLineStart = 0;
     this.m_curLineEnd = 0;
     this.m_numBytesWritten = 0;
     this.m_calcFP = false;
     this.m_isClosed = false;
     this.m_isASCIIOrUTF8 = (encoding == Encoding.UTF8 || encoding == Encoding.ASCII);
 }
Пример #5
0
 internal HpcBinaryWriter(NativeBlockStream stream, Encoding encoding, Int32 buffSize)
 {
     this.m_nativeStream = stream;
     this.m_encoding = encoding;
     this.m_nextBlockSize = Math.Max(DefaultBlockSize, buffSize / 2);
     this.m_bufferSizeHint = buffSize;
     this.m_curDataBlockInfo.dataBlock = null;
     this.m_curDataBlockInfo.blockSize = 0;
     this.m_curDataBlockInfo.itemHandle = IntPtr.Zero;
     this.m_curDataBlock = this.m_curDataBlockInfo.dataBlock;
     this.m_curBlockSize = this.m_curDataBlockInfo.blockSize;
     this.m_curRecordStart = 0;
     this.m_curRecordEnd = 0;
     this.m_charMaxByteCount = this.m_encoding.GetMaxByteCount(1);
     this.m_isClosed = false;
     this.m_numBytesWritten = 0;
     this.m_calcFP = false;
     this.m_bfm = new BinaryFormatter();
 }
Пример #6
0
 public override HpcRecordReader <ushort> MakeReader(NativeBlockStream nativeStream)
 {
     return(new HpcRecordUShortReader(new HpcBinaryReader(nativeStream)));
 }
Пример #7
0
 public override HpcRecordReader <int> MakeReader(NativeBlockStream nativeStream)
 {
     return(new HpcRecordInt32Reader(new HpcBinaryReader(nativeStream)));
 }
Пример #8
0
 /// <summary>
 /// Initializes an instance of DryadLinqTextWriter with encoding UTF8.
 /// </summary>
 /// <param name="stream">A native stream to write to.</param>
 public DryadLinqTextWriter(NativeBlockStream stream)
     : this(stream, Encoding.UTF8)
 {
 }
Пример #9
0
 public override HpcRecordReader <char> MakeReader(NativeBlockStream nativeStream)
 {
     return(new HpcRecordCharReader(new HpcBinaryReader(nativeStream)));
 }
Пример #10
0
 public override DryadLinqRecordReader <decimal> MakeReader(NativeBlockStream nativeStream)
 {
     return(new DryadLinqRecordDecimalReader(new DryadLinqBinaryReader(nativeStream)));
 }
Пример #11
0
 public override DryadLinqRecordWriter <LineRecord> MakeWriter(NativeBlockStream nativeStream)
 {
     return(new DryadLinqRecordTextWriter(new DryadLinqTextWriter(nativeStream)));
 }
Пример #12
0
 public abstract void Write(NativeBlockStream stream);
Пример #13
0
 public override DryadLinqRecordReader <float> MakeReader(NativeBlockStream nativeStream)
 {
     return(new DryadLinqRecordFloatReader(new DryadLinqBinaryReader(nativeStream)));
 }
Пример #14
0
 public override DryadLinqRecordReader <sbyte> MakeReader(NativeBlockStream nativeStream)
 {
     return(new DryadLinqRecordSByteReader(new DryadLinqBinaryReader(nativeStream)));
 }
Пример #15
0
 public static HpcBinaryWriter MakeBinaryWriter(NativeBlockStream nativeStream)
 {
     return(new HpcBinaryWriter(nativeStream));
 }
Пример #16
0
 public override DryadLinqRecordWriter <byte> MakeWriter(NativeBlockStream nativeStream)
 {
     return(new DryadLinqRecordByteWriter(new DryadLinqBinaryWriter(nativeStream)));
 }
Пример #17
0
 /// <summary>
 /// Creates a new writer from a native stream.
 /// </summary>
 /// <param name="nativeStream">A native stream.</param>
 /// <returns>A writer that uses DryadLINQ serialization for record of type T.</returns>
 public abstract DryadLinqRecordWriter <T> MakeWriter(NativeBlockStream nativeStream);
Пример #18
0
 public override DryadLinqRecordWriter <SqlDateTime> MakeWriter(NativeBlockStream nativeStream)
 {
     return(new DryadLinqRecordSqlDateTimeWriter(new DryadLinqBinaryWriter(nativeStream)));
 }
Пример #19
0
 public override HpcRecordReader <ulong> MakeReader(NativeBlockStream nativeStream)
 {
     return(new HpcRecordUInt64Reader(new HpcBinaryReader(nativeStream)));
 }
Пример #20
0
 /// <summary>
 /// Initializes an instance of DryadLinqTextWriter.
 /// </summary>
 /// <param name="stream">A native stream to write to.</param>
 /// <param name="encoding">The text encoding.</param>
 public DryadLinqTextWriter(NativeBlockStream stream, Encoding encoding)
     : this(stream, encoding, DefaultBlockSize)
 {
 }
Пример #21
0
 public override DryadLinqRecordReader <DateTime> MakeReader(NativeBlockStream nativeStream)
 {
     return(new DryadLinqRecordDateTimeReader(new DryadLinqBinaryReader(nativeStream)));
 }
Пример #22
0
 /// <summary>
 /// Make a binary writer from a native stream. Used only by auto-generated code.
 /// </summary>
 /// <param name="nativeStream">A native stream</param>
 /// <returns>A binary writer</returns>
 public static DryadLinqBinaryWriter MakeBinaryWriter(NativeBlockStream nativeStream)
 {
     return(new DryadLinqBinaryWriter(nativeStream));
 }
Пример #23
0
 internal DryadLinqBinaryReader(NativeBlockStream stream)
     : this(stream, Encoding.UTF8)
 {
 }
Пример #24
0
 internal HpcBinaryWriter(NativeBlockStream stream, Encoding encoding)
     : this(stream, encoding, DefaultBlockSize)
 {
 }
Пример #25
0
 public HpcTextReader(NativeBlockStream stream)
     : this(stream, Encoding.UTF8)
 {
 }
Пример #26
0
 /// <summary>
 /// Creates a new reader from a native stream.
 /// </summary>
 /// <param name="nativeStream">A native stream.</param>
 /// <returns>A reader that uses DryadLINQ serialization for records of type T.</returns>
 public abstract DryadLinqRecordReader <T> MakeReader(NativeBlockStream nativeStream);
Пример #27
0
 internal HpcBinaryWriter(NativeBlockStream stream)
     : this(stream, Encoding.UTF8)
 {
 }
Пример #28
0
        internal override unsafe DataBlockInfo ReadDataBlock()
        {
            // free the dummy block if it was allocated.
            if (s_newlineByteBlock != null)
            {
                Marshal.FreeHGlobal((IntPtr)s_newlineByteBlock);
                s_newlineByteBlock = null;
            }

            while (true)
            {
                DataBlockInfo dataBlockInfo = this.m_curStream.ReadDataBlock();

                if (dataBlockInfo.blockSize == 0 && this.m_curIdx == m_srcList.Count)
                {
                    // data has been exhausted.  We return the empty block and the caller knows what to do.
                    return(dataBlockInfo);
                }

                // normal case.. record the last two bytes for newline tracking, and return the block.
                if (dataBlockInfo.blockSize > 0)
                {
                    if (m_appendNewLinesToFiles)
                    {
                        if (dataBlockInfo.blockSize >= 2)
                        {
                            m_trailingBytesOfData[0] = dataBlockInfo.dataBlock[dataBlockInfo.blockSize - 2];
                            m_trailingBytesOfData[1] = dataBlockInfo.dataBlock[dataBlockInfo.blockSize - 1];
                        }
                        else
                        {
                            Debug.Assert(dataBlockInfo.blockSize == 1);
                            // CASE: dataBlockInfo.blockSize == 1
                            // shift left.
                            // We must do this otherwise the following data could fail to be identified
                            //    Blocks = [.........\r] [\n]
                            m_trailingBytesOfData[0] = m_trailingBytesOfData[1];   //shift
                            m_trailingBytesOfData[1] = dataBlockInfo.dataBlock[0]; // record the single element.
                        }
                    }

                    return(dataBlockInfo);
                }

                this.m_curStream.ReleaseDataBlock(dataBlockInfo.itemHandle);
                this.m_curStream.Close();
                this.m_curStream = this.GetStream(this.m_curIdx++, FileAccess.Read);

                // we only get here when a file is fully consumed and it wasn't the last file in the set.
                // if the data stream didn't end with a newline-pair, emit one so that
                // LineRecord-parsing will work correctly.
                // the next time we enter this method, we will free the unmanaged data and the
                // next real block of data will be read and consumed.

                if (m_appendNewLinesToFiles)
                {
                    //@@TODO[p3]: we currently only observe and insert \r\n pairs.
                    //            Unicode may have other types of newline to consider.
                    if (m_trailingBytesOfData[0] != '\r' || m_trailingBytesOfData[1] != '\n')
                    {
                        // create the dummy block.
                        if (s_newlineByteBlock == null)
                        {
                            s_newlineByteBlock    = (byte *)Marshal.AllocHGlobal(2);
                            s_newlineByteBlock[0] = (byte)'\r';
                            s_newlineByteBlock[1] = (byte)'\n';
                        }
                        DataBlockInfo dummyblock = new DataBlockInfo();
                        dummyblock.blockSize  = 2;
                        dummyblock.dataBlock  = s_newlineByteBlock;
                        dummyblock.itemHandle = IntPtr.Zero;

                        return(dummyblock);
                    }
                }
            }
        }
Пример #29
0
 public abstract void Write(NativeBlockStream stream);
Пример #30
0
        private NativeBlockStream m_nativeStream; // source stream

        #endregion Fields

        #region Constructors

        public HpcTextReader(NativeBlockStream stream)
            : this(stream, Encoding.UTF8)
        {
        }
Пример #31
0
 /// <summary>
 /// Initializes an instance of DryadLinqTextWriter.
 /// </summary>
 /// <param name="stream">A native stream to write to.</param>
 /// <param name="encoding">The text encoding.</param>
 public DryadLinqTextWriter(NativeBlockStream stream, Encoding encoding)
     : this(stream, encoding, DefaultBlockSize)
 {
 }
Пример #32
0
        internal unsafe override DataBlockInfo ReadDataBlock()
        {
            // free the dummy block if it was allocated.
            if (s_newlineByteBlock != null)
            {
                Marshal.FreeHGlobal((IntPtr) s_newlineByteBlock);
                s_newlineByteBlock = null;
            }

            while (true)
            {
                DataBlockInfo dataBlockInfo = this.m_curStream.ReadDataBlock();

                if (dataBlockInfo.blockSize == 0 && this.m_curIdx == m_srcList.Count)
                {
                    // data has been exhausted.  We return the empty block and the caller knows what to do.
                    return dataBlockInfo;
                }

                // normal case.. record the last two bytes for newline tracking, and return the block.
                if (dataBlockInfo.blockSize > 0)
                {
                    if (m_appendNewLinesToFiles)
                    {
                        if (dataBlockInfo.blockSize >= 2)
                        {
                            m_trailingBytesOfData[0] = dataBlockInfo.dataBlock[dataBlockInfo.blockSize - 2];
                            m_trailingBytesOfData[1] = dataBlockInfo.dataBlock[dataBlockInfo.blockSize - 1];
                        }
                        else
                        {
                            Debug.Assert(dataBlockInfo.blockSize == 1);
                            // CASE: dataBlockInfo.blockSize == 1
                            // shift left.
                            // We must do this otherwise the following data could fail to be identified
                            //    Blocks = [.........\r] [\n]
                            m_trailingBytesOfData[0] = m_trailingBytesOfData[1]; //shift
                            m_trailingBytesOfData[1] = dataBlockInfo.dataBlock[0]; // record the single element.
                        }
                    }

                    return dataBlockInfo;
                }

                this.m_curStream.ReleaseDataBlock(dataBlockInfo.itemHandle);
                this.m_curStream.Close();
                this.m_curStream = this.GetStream(this.m_curIdx++, FileAccess.Read);

                // we only get here when a file is fully consumed and it wasn't the last file in the set.
                // if the data stream didn't end with a newline-pair, emit one so that
                // LineRecord-parsing will work correctly.
                // the next time we enter this method, we will free the unmanaged data and the
                // next real block of data will be read and consumed.

                if (m_appendNewLinesToFiles)
                {
                    //@@TODO[p3]: we currently only observe and insert \r\n pairs.
                    //            Unicode may have other types of newline to consider.
                    if (m_trailingBytesOfData[0] != '\r' || m_trailingBytesOfData[1] != '\n')
                    {
                        // create the dummy block.
                        if (s_newlineByteBlock == null)
                        {
                            s_newlineByteBlock = (byte*)Marshal.AllocHGlobal(2);
                            s_newlineByteBlock[0] = (byte)'\r';
                            s_newlineByteBlock[1] = (byte)'\n';
                        }
                        DataBlockInfo dummyblock = new DataBlockInfo();
                        dummyblock.blockSize = 2;
                        dummyblock.dataBlock = s_newlineByteBlock;
                        dummyblock.itemHandle = IntPtr.Zero;

                        return dummyblock;
                    }
                }
            }
        }
Пример #33
0
 public override HpcRecordWriter <char> MakeWriter(NativeBlockStream nativeStream)
 {
     return(new HpcRecordCharWriter(new HpcBinaryWriter(nativeStream)));
 }
Пример #34
0
 /// <summary>
 /// Initializes an instance of DryadLinqTextReader with encoding UTF8.
 /// </summary>
 /// <param name="stream">A native stream to read from.</param>
 public DryadLinqTextReader(NativeBlockStream stream)
     : this(stream, Encoding.UTF8)
 {
 }
Пример #35
0
 public override HpcRecordWriter <ushort> MakeWriter(NativeBlockStream nativeStream)
 {
     return(new HpcRecordUShortWriter(new HpcBinaryWriter(nativeStream)));
 }
Пример #36
0
 public override HpcRecordReader <float> MakeReader(NativeBlockStream nativeStream)
 {
     return(new HpcRecordFloatReader(new HpcBinaryReader(nativeStream)));
 }
Пример #37
0
 public override HpcRecordWriter <int> MakeWriter(NativeBlockStream nativeStream)
 {
     return(new HpcRecordInt32Writer(new HpcBinaryWriter(nativeStream)));
 }
Пример #38
0
 public override HpcRecordReader <decimal> MakeReader(NativeBlockStream nativeStream)
 {
     return(new HpcRecordDecimalReader(new HpcBinaryReader(nativeStream)));
 }
Пример #39
0
 public override HpcRecordWriter <ulong> MakeWriter(NativeBlockStream nativeStream)
 {
     return(new HpcRecordUInt64Writer(new HpcBinaryWriter(nativeStream)));
 }
Пример #40
0
 public override DryadLinqRecordWriter <decimal> MakeWriter(NativeBlockStream nativeStream)
 {
     return(new DryadLinqRecordDecimalWriter(new DryadLinqBinaryWriter(nativeStream)));
 }
Пример #41
0
 public override HpcRecordWriter <float> MakeWriter(NativeBlockStream nativeStream)
 {
     return(new HpcRecordFloatWriter(new HpcBinaryWriter(nativeStream)));
 }
Пример #42
0
        internal override unsafe DataBlockInfo ReadDataBlock()
        {
            while (true)
            {
                DataBlockInfo dataBlockInfo = this.m_curStream.ReadDataBlock();
                if (dataBlockInfo.BlockSize == 0 && this.m_curIdx == m_srcList.Count)
                {
                    // data has been exhausted.  We return the empty block and the caller knows what to do.
                    return dataBlockInfo;
                }
                
                // normal case.. record the last two bytes for newline tracking, and return the block.
                if (dataBlockInfo.BlockSize > 0)
                {
                    return dataBlockInfo;
                }

                this.m_curStream.ReleaseDataBlock(dataBlockInfo.ItemHandle);
                this.m_curStream.Close();
                this.m_curStream = this.GetStream(this.m_curIdx++, FileAccess.Read);
            }
        }
Пример #43
0
 public override HpcRecordWriter <decimal> MakeWriter(NativeBlockStream nativeStream)
 {
     return(new HpcRecordDecimalWriter(new HpcBinaryWriter(nativeStream)));
 }
Пример #44
0
 public override DryadLinqRecordWriter <ulong> MakeWriter(NativeBlockStream nativeStream)
 {
     return(new DryadLinqRecordUInt64Writer(new DryadLinqBinaryWriter(nativeStream)));
 }