internal DryadLinqBinaryReader(NativeBlockStream stream, Encoding encoding) { this.m_nativeStream = stream; this.m_encoding = encoding; this.m_decoder = encoding.GetDecoder(); this.m_curDataBlockInfo.DataBlock = null; this.m_curDataBlockInfo.BlockSize = -1; this.m_curDataBlockInfo.ItemHandle = IntPtr.Zero; this.m_curDataBlock = this.m_curDataBlockInfo.DataBlock; this.m_curBlockSize = this.m_curDataBlockInfo.BlockSize; this.m_curBlockPos = -1; this.m_isClosed = false; }
/// <summary> /// Initializes an instance of DryadLinqTextReader. /// </summary> /// <param name="stream">A native stream to read from.</param> /// <param name="encoding">The text encoding.</param> public DryadLinqTextReader(NativeBlockStream stream, Encoding encoding) { this.m_nativeStream = stream; this.m_encoding = encoding; this.m_decoder = encoding.GetDecoder(); this.m_curDataBlockInfo.DataBlock = null; this.m_curDataBlockInfo.BlockSize = -1; this.m_curDataBlockInfo.ItemHandle = IntPtr.Zero; this.m_curBlockPos = 0; this.m_decodeUnitCharSize = this.m_encoding.GetMaxCharCount(DecodeUnitByteSize); this.m_charBuff = new char[this.m_decodeUnitCharSize + 2]; //allow 2 bytes for trailing newline this.m_charBuffEnd = 0; this.m_curLineStart = 0; this.m_curLineEnd = 0; this.m_isClosed = false; }
internal MultiBlockStream(List<string[]> srcList, string associatedDscStreamName, FileAccess access, CompressionScheme scheme) { this.m_srcList = srcList; this.m_associatedDscStreamName = associatedDscStreamName; if (srcList.Count == 0) { throw new DryadLinqException(DryadLinqErrorCode.MultiBlockEmptyPartitionList, SR.MultiBlockEmptyPartitionList); } this.m_compressionScheme = scheme; this.m_curIdx = 0; this.m_curStream = this.GetStream(this.m_curIdx++, access); }
/// <summary> /// Initializes an instance of DryadLinqTextWriter. /// </summary> /// <param name="stream">A native stream to write to.</param> /// <param name="encoding">The text encoding.</param> /// <param name="buffSize">A hint for the size of write buffer.</param> public DryadLinqTextWriter(NativeBlockStream stream, Encoding encoding, Int32 buffSize) { this.m_nativeStream = stream; this.m_encoding = encoding; this.m_nextBlockSize = Math.Max(DefaultBlockSize, buffSize/2); this.m_bufferSizeHint = buffSize; this.m_curDataBlockInfo.DataBlock = null; this.m_curDataBlockInfo.BlockSize = 0; this.m_curDataBlockInfo.ItemHandle = IntPtr.Zero; this.m_curDataBlock = this.m_curDataBlockInfo.DataBlock; this.m_curBlockSize = this.m_curDataBlockInfo.BlockSize; this.m_curLineStart = 0; this.m_curLineEnd = 0; this.m_numBytesWritten = 0; this.m_calcFP = false; this.m_isClosed = false; this.m_isASCIIOrUTF8 = (encoding == Encoding.UTF8 || encoding == Encoding.ASCII); }
internal HpcBinaryWriter(NativeBlockStream stream, Encoding encoding, Int32 buffSize) { this.m_nativeStream = stream; this.m_encoding = encoding; this.m_nextBlockSize = Math.Max(DefaultBlockSize, buffSize / 2); this.m_bufferSizeHint = buffSize; this.m_curDataBlockInfo.dataBlock = null; this.m_curDataBlockInfo.blockSize = 0; this.m_curDataBlockInfo.itemHandle = IntPtr.Zero; this.m_curDataBlock = this.m_curDataBlockInfo.dataBlock; this.m_curBlockSize = this.m_curDataBlockInfo.blockSize; this.m_curRecordStart = 0; this.m_curRecordEnd = 0; this.m_charMaxByteCount = this.m_encoding.GetMaxByteCount(1); this.m_isClosed = false; this.m_numBytesWritten = 0; this.m_calcFP = false; this.m_bfm = new BinaryFormatter(); }
public override HpcRecordReader <ushort> MakeReader(NativeBlockStream nativeStream) { return(new HpcRecordUShortReader(new HpcBinaryReader(nativeStream))); }
public override HpcRecordReader <int> MakeReader(NativeBlockStream nativeStream) { return(new HpcRecordInt32Reader(new HpcBinaryReader(nativeStream))); }
/// <summary> /// Initializes an instance of DryadLinqTextWriter with encoding UTF8. /// </summary> /// <param name="stream">A native stream to write to.</param> public DryadLinqTextWriter(NativeBlockStream stream) : this(stream, Encoding.UTF8) { }
public override HpcRecordReader <char> MakeReader(NativeBlockStream nativeStream) { return(new HpcRecordCharReader(new HpcBinaryReader(nativeStream))); }
public override DryadLinqRecordReader <decimal> MakeReader(NativeBlockStream nativeStream) { return(new DryadLinqRecordDecimalReader(new DryadLinqBinaryReader(nativeStream))); }
public override DryadLinqRecordWriter <LineRecord> MakeWriter(NativeBlockStream nativeStream) { return(new DryadLinqRecordTextWriter(new DryadLinqTextWriter(nativeStream))); }
public abstract void Write(NativeBlockStream stream);
public override DryadLinqRecordReader <float> MakeReader(NativeBlockStream nativeStream) { return(new DryadLinqRecordFloatReader(new DryadLinqBinaryReader(nativeStream))); }
public override DryadLinqRecordReader <sbyte> MakeReader(NativeBlockStream nativeStream) { return(new DryadLinqRecordSByteReader(new DryadLinqBinaryReader(nativeStream))); }
public static HpcBinaryWriter MakeBinaryWriter(NativeBlockStream nativeStream) { return(new HpcBinaryWriter(nativeStream)); }
public override DryadLinqRecordWriter <byte> MakeWriter(NativeBlockStream nativeStream) { return(new DryadLinqRecordByteWriter(new DryadLinqBinaryWriter(nativeStream))); }
/// <summary> /// Creates a new writer from a native stream. /// </summary> /// <param name="nativeStream">A native stream.</param> /// <returns>A writer that uses DryadLINQ serialization for record of type T.</returns> public abstract DryadLinqRecordWriter <T> MakeWriter(NativeBlockStream nativeStream);
public override DryadLinqRecordWriter <SqlDateTime> MakeWriter(NativeBlockStream nativeStream) { return(new DryadLinqRecordSqlDateTimeWriter(new DryadLinqBinaryWriter(nativeStream))); }
public override HpcRecordReader <ulong> MakeReader(NativeBlockStream nativeStream) { return(new HpcRecordUInt64Reader(new HpcBinaryReader(nativeStream))); }
/// <summary> /// Initializes an instance of DryadLinqTextWriter. /// </summary> /// <param name="stream">A native stream to write to.</param> /// <param name="encoding">The text encoding.</param> public DryadLinqTextWriter(NativeBlockStream stream, Encoding encoding) : this(stream, encoding, DefaultBlockSize) { }
public override DryadLinqRecordReader <DateTime> MakeReader(NativeBlockStream nativeStream) { return(new DryadLinqRecordDateTimeReader(new DryadLinqBinaryReader(nativeStream))); }
/// <summary> /// Make a binary writer from a native stream. Used only by auto-generated code. /// </summary> /// <param name="nativeStream">A native stream</param> /// <returns>A binary writer</returns> public static DryadLinqBinaryWriter MakeBinaryWriter(NativeBlockStream nativeStream) { return(new DryadLinqBinaryWriter(nativeStream)); }
internal DryadLinqBinaryReader(NativeBlockStream stream) : this(stream, Encoding.UTF8) { }
internal HpcBinaryWriter(NativeBlockStream stream, Encoding encoding) : this(stream, encoding, DefaultBlockSize) { }
public HpcTextReader(NativeBlockStream stream) : this(stream, Encoding.UTF8) { }
/// <summary> /// Creates a new reader from a native stream. /// </summary> /// <param name="nativeStream">A native stream.</param> /// <returns>A reader that uses DryadLINQ serialization for records of type T.</returns> public abstract DryadLinqRecordReader <T> MakeReader(NativeBlockStream nativeStream);
internal HpcBinaryWriter(NativeBlockStream stream) : this(stream, Encoding.UTF8) { }
internal override unsafe DataBlockInfo ReadDataBlock() { // free the dummy block if it was allocated. if (s_newlineByteBlock != null) { Marshal.FreeHGlobal((IntPtr)s_newlineByteBlock); s_newlineByteBlock = null; } while (true) { DataBlockInfo dataBlockInfo = this.m_curStream.ReadDataBlock(); if (dataBlockInfo.blockSize == 0 && this.m_curIdx == m_srcList.Count) { // data has been exhausted. We return the empty block and the caller knows what to do. return(dataBlockInfo); } // normal case.. record the last two bytes for newline tracking, and return the block. if (dataBlockInfo.blockSize > 0) { if (m_appendNewLinesToFiles) { if (dataBlockInfo.blockSize >= 2) { m_trailingBytesOfData[0] = dataBlockInfo.dataBlock[dataBlockInfo.blockSize - 2]; m_trailingBytesOfData[1] = dataBlockInfo.dataBlock[dataBlockInfo.blockSize - 1]; } else { Debug.Assert(dataBlockInfo.blockSize == 1); // CASE: dataBlockInfo.blockSize == 1 // shift left. // We must do this otherwise the following data could fail to be identified // Blocks = [.........\r] [\n] m_trailingBytesOfData[0] = m_trailingBytesOfData[1]; //shift m_trailingBytesOfData[1] = dataBlockInfo.dataBlock[0]; // record the single element. } } return(dataBlockInfo); } this.m_curStream.ReleaseDataBlock(dataBlockInfo.itemHandle); this.m_curStream.Close(); this.m_curStream = this.GetStream(this.m_curIdx++, FileAccess.Read); // we only get here when a file is fully consumed and it wasn't the last file in the set. // if the data stream didn't end with a newline-pair, emit one so that // LineRecord-parsing will work correctly. // the next time we enter this method, we will free the unmanaged data and the // next real block of data will be read and consumed. if (m_appendNewLinesToFiles) { //@@TODO[p3]: we currently only observe and insert \r\n pairs. // Unicode may have other types of newline to consider. if (m_trailingBytesOfData[0] != '\r' || m_trailingBytesOfData[1] != '\n') { // create the dummy block. if (s_newlineByteBlock == null) { s_newlineByteBlock = (byte *)Marshal.AllocHGlobal(2); s_newlineByteBlock[0] = (byte)'\r'; s_newlineByteBlock[1] = (byte)'\n'; } DataBlockInfo dummyblock = new DataBlockInfo(); dummyblock.blockSize = 2; dummyblock.dataBlock = s_newlineByteBlock; dummyblock.itemHandle = IntPtr.Zero; return(dummyblock); } } } }
private NativeBlockStream m_nativeStream; // source stream #endregion Fields #region Constructors public HpcTextReader(NativeBlockStream stream) : this(stream, Encoding.UTF8) { }
internal unsafe override DataBlockInfo ReadDataBlock() { // free the dummy block if it was allocated. if (s_newlineByteBlock != null) { Marshal.FreeHGlobal((IntPtr) s_newlineByteBlock); s_newlineByteBlock = null; } while (true) { DataBlockInfo dataBlockInfo = this.m_curStream.ReadDataBlock(); if (dataBlockInfo.blockSize == 0 && this.m_curIdx == m_srcList.Count) { // data has been exhausted. We return the empty block and the caller knows what to do. return dataBlockInfo; } // normal case.. record the last two bytes for newline tracking, and return the block. if (dataBlockInfo.blockSize > 0) { if (m_appendNewLinesToFiles) { if (dataBlockInfo.blockSize >= 2) { m_trailingBytesOfData[0] = dataBlockInfo.dataBlock[dataBlockInfo.blockSize - 2]; m_trailingBytesOfData[1] = dataBlockInfo.dataBlock[dataBlockInfo.blockSize - 1]; } else { Debug.Assert(dataBlockInfo.blockSize == 1); // CASE: dataBlockInfo.blockSize == 1 // shift left. // We must do this otherwise the following data could fail to be identified // Blocks = [.........\r] [\n] m_trailingBytesOfData[0] = m_trailingBytesOfData[1]; //shift m_trailingBytesOfData[1] = dataBlockInfo.dataBlock[0]; // record the single element. } } return dataBlockInfo; } this.m_curStream.ReleaseDataBlock(dataBlockInfo.itemHandle); this.m_curStream.Close(); this.m_curStream = this.GetStream(this.m_curIdx++, FileAccess.Read); // we only get here when a file is fully consumed and it wasn't the last file in the set. // if the data stream didn't end with a newline-pair, emit one so that // LineRecord-parsing will work correctly. // the next time we enter this method, we will free the unmanaged data and the // next real block of data will be read and consumed. if (m_appendNewLinesToFiles) { //@@TODO[p3]: we currently only observe and insert \r\n pairs. // Unicode may have other types of newline to consider. if (m_trailingBytesOfData[0] != '\r' || m_trailingBytesOfData[1] != '\n') { // create the dummy block. if (s_newlineByteBlock == null) { s_newlineByteBlock = (byte*)Marshal.AllocHGlobal(2); s_newlineByteBlock[0] = (byte)'\r'; s_newlineByteBlock[1] = (byte)'\n'; } DataBlockInfo dummyblock = new DataBlockInfo(); dummyblock.blockSize = 2; dummyblock.dataBlock = s_newlineByteBlock; dummyblock.itemHandle = IntPtr.Zero; return dummyblock; } } } }
public override HpcRecordWriter <char> MakeWriter(NativeBlockStream nativeStream) { return(new HpcRecordCharWriter(new HpcBinaryWriter(nativeStream))); }
/// <summary> /// Initializes an instance of DryadLinqTextReader with encoding UTF8. /// </summary> /// <param name="stream">A native stream to read from.</param> public DryadLinqTextReader(NativeBlockStream stream) : this(stream, Encoding.UTF8) { }
public override HpcRecordWriter <ushort> MakeWriter(NativeBlockStream nativeStream) { return(new HpcRecordUShortWriter(new HpcBinaryWriter(nativeStream))); }
public override HpcRecordReader <float> MakeReader(NativeBlockStream nativeStream) { return(new HpcRecordFloatReader(new HpcBinaryReader(nativeStream))); }
public override HpcRecordWriter <int> MakeWriter(NativeBlockStream nativeStream) { return(new HpcRecordInt32Writer(new HpcBinaryWriter(nativeStream))); }
public override HpcRecordReader <decimal> MakeReader(NativeBlockStream nativeStream) { return(new HpcRecordDecimalReader(new HpcBinaryReader(nativeStream))); }
public override HpcRecordWriter <ulong> MakeWriter(NativeBlockStream nativeStream) { return(new HpcRecordUInt64Writer(new HpcBinaryWriter(nativeStream))); }
public override DryadLinqRecordWriter <decimal> MakeWriter(NativeBlockStream nativeStream) { return(new DryadLinqRecordDecimalWriter(new DryadLinqBinaryWriter(nativeStream))); }
public override HpcRecordWriter <float> MakeWriter(NativeBlockStream nativeStream) { return(new HpcRecordFloatWriter(new HpcBinaryWriter(nativeStream))); }
internal override unsafe DataBlockInfo ReadDataBlock() { while (true) { DataBlockInfo dataBlockInfo = this.m_curStream.ReadDataBlock(); if (dataBlockInfo.BlockSize == 0 && this.m_curIdx == m_srcList.Count) { // data has been exhausted. We return the empty block and the caller knows what to do. return dataBlockInfo; } // normal case.. record the last two bytes for newline tracking, and return the block. if (dataBlockInfo.BlockSize > 0) { return dataBlockInfo; } this.m_curStream.ReleaseDataBlock(dataBlockInfo.ItemHandle); this.m_curStream.Close(); this.m_curStream = this.GetStream(this.m_curIdx++, FileAccess.Read); } }
public override HpcRecordWriter <decimal> MakeWriter(NativeBlockStream nativeStream) { return(new HpcRecordDecimalWriter(new HpcBinaryWriter(nativeStream))); }
public override DryadLinqRecordWriter <ulong> MakeWriter(NativeBlockStream nativeStream) { return(new DryadLinqRecordUInt64Writer(new DryadLinqBinaryWriter(nativeStream))); }