private void FlushDataBlock() { DataBlockInfo newDataBlockInfo; if (this.m_curLineStart == 0) { // The current block is too small for a single record, augment it if (this.m_curBlockSize == this.m_nextBlockSize) { throw new DryadLinqException(DryadLinqErrorCode.RecordSizeMax2GB, SR.RecordSizeMax2GB); } newDataBlockInfo = this.m_nativeStream.AllocateDataBlock(this.m_nextBlockSize); this.m_nextBlockSize = this.m_nextBlockSize * 2; if (this.m_nextBlockSize < 0) { this.m_nextBlockSize = 0x7FFFFFF8; } this.m_nativeStream.ReleaseDataBlock(this.m_curDataBlockInfo.ItemHandle); this.m_curDataBlockInfo.ItemHandle = IntPtr.Zero; } else { // Write all the complete records in the block this.m_nativeStream.WriteDataBlock(this.m_curDataBlockInfo.ItemHandle, this.m_curLineStart); this.m_numBytesWritten += this.m_curLineStart; this.m_nativeStream.ReleaseDataBlock(this.m_curDataBlockInfo.ItemHandle); this.m_curDataBlockInfo.ItemHandle = IntPtr.Zero; newDataBlockInfo = this.m_nativeStream.AllocateDataBlock(this.m_curBlockSize); this.m_curLineEnd -= this.m_curLineStart; this.m_curLineStart = 0; } this.m_curDataBlockInfo = newDataBlockInfo; this.m_curDataBlock = newDataBlockInfo.DataBlock; this.m_curBlockSize = newDataBlockInfo.BlockSize; }
// Get the next data block private unsafe void GetNextDataBlock() { this.m_nativeStream.ReleaseDataBlock(this.m_curDataBlockInfo.ItemHandle); this.m_curDataBlockInfo.ItemHandle = IntPtr.Zero; this.m_curDataBlockInfo = this.m_nativeStream.ReadDataBlock(); this.m_curBlockPos = 0; }
/// <summary> /// Flushes the current write buffer. /// </summary> public void Flush() { Debug.Assert(this.m_curLineStart == this.m_curLineEnd); if (this.m_curLineStart > 0) { this.m_nativeStream.WriteDataBlock(this.m_curDataBlockInfo.ItemHandle, this.m_curLineStart); this.m_numBytesWritten += this.m_curLineStart; this.m_nativeStream.ReleaseDataBlock(this.m_curDataBlockInfo.ItemHandle); this.m_curDataBlockInfo.ItemHandle = IntPtr.Zero; this.m_curDataBlockInfo = this.m_nativeStream.AllocateDataBlock(this.m_curBlockSize); this.m_curDataBlock = this.m_curDataBlockInfo.DataBlock; this.m_curBlockSize = this.m_curDataBlockInfo.BlockSize; this.m_curLineStart = 0; this.m_curLineEnd = 0; } this.m_nativeStream.Flush(); }
internal override unsafe DataBlockInfo ReadDataBlock() { while (true) { DataBlockInfo dataBlockInfo = this.m_curStream.ReadDataBlock(); if (dataBlockInfo.BlockSize == 0 && this.m_curIdx == m_srcList.Count) { // data has been exhausted. We return the empty block and the caller knows what to do. return(dataBlockInfo); } // normal case.. record the last two bytes for newline tracking, and return the block. if (dataBlockInfo.BlockSize > 0) { return(dataBlockInfo); } this.m_curStream.ReleaseDataBlock(dataBlockInfo.ItemHandle); this.m_curStream.Close(); this.m_curStream = this.GetStream(this.m_curIdx++, FileAccess.Read); } }
/// <summary> /// Private helper to write the current block out to the native stream. /// - it writes out the current data buffer up to the point it was filled /// - it releases the current data block back to the native stream code (which owns the lifecycle of read buffers), /// - then allocated a new buffer from the native stream /// - and updates the internal read buffer pointer and position members /// </summary> private void FlushDataBlock() { DataBlockInfo newDataBlockInfo; if (this.m_curRecordStart <= 16) { // The current block is too small for a single record, augment it if (this.m_curBlockSize == this.m_nextBlockSize) { throw new DryadLinqException(HpcLinqErrorCode.RecordSizeMax2GB, SR.RecordSizeMax2GB); } newDataBlockInfo = this.m_nativeStream.AllocateDataBlock(this.m_nextBlockSize); this.m_nextBlockSize = this.m_nextBlockSize * 2; if (this.m_nextBlockSize < 0) { this.m_nextBlockSize = 0x7FFFFFF8; } HpcLinqUtil.memcpy(this.m_curDataBlock, newDataBlockInfo.dataBlock, this.m_curRecordEnd); } else { // Write all the complete records in the block, put the partial record in the new block newDataBlockInfo = this.m_nativeStream.AllocateDataBlock(this.m_curBlockSize); HpcLinqUtil.memcpy(this.m_curDataBlock + this.m_curRecordStart, newDataBlockInfo.dataBlock, this.m_curRecordEnd - this.m_curRecordStart); this.m_nativeStream.WriteDataBlock(this.m_curDataBlockInfo.itemHandle, this.m_curRecordStart); this.m_numBytesWritten += this.m_curRecordStart; this.m_curRecordEnd -= this.m_curRecordStart; this.m_curRecordStart = 0; } this.m_nativeStream.ReleaseDataBlock(this.m_curDataBlockInfo.itemHandle); this.m_curDataBlockInfo.itemHandle = IntPtr.Zero; this.m_curDataBlockInfo = newDataBlockInfo; this.m_curDataBlock = newDataBlockInfo.dataBlock; this.m_curBlockSize = newDataBlockInfo.blockSize; }
/// <summary> /// Writes out the current data buffer (equivalent of FlushDataBlock), and calls /// Flush on the native stream to ensure all the data makes its way to the disk /// </summary> internal void Flush() { if (this.m_curRecordEnd > 0) { this.m_nativeStream.WriteDataBlock(this.m_curDataBlockInfo.itemHandle, this.m_curRecordEnd); this.m_numBytesWritten += this.m_curRecordEnd; this.m_nativeStream.ReleaseDataBlock(this.m_curDataBlockInfo.itemHandle); this.m_curDataBlockInfo.itemHandle = IntPtr.Zero; this.m_curDataBlockInfo = this.m_nativeStream.AllocateDataBlock(this.m_curBlockSize); this.m_curDataBlock = this.m_curDataBlockInfo.dataBlock; this.m_curBlockSize = this.m_curDataBlockInfo.blockSize; this.m_curRecordStart = 0; this.m_curRecordEnd = 0; } this.m_nativeStream.Flush(); }
/// <summary> /// Private helper to request a new data block from the native stream. /// - it releases the current data block back to the native stream code /// (which owns the lifecycle of read buffers), /// - then requests a new buffer from the native stream /// - and updates the internal read buffer pointer (m_curDataBlock), size (m_curDataSize) /// and position, all of which are needed by subsequent Read*() calls coming from the user /// </summary> private unsafe void GetNextDataBlock() { this.m_nativeStream.ReleaseDataBlock(this.m_curDataBlockInfo.ItemHandle); this.m_curDataBlockInfo.ItemHandle = IntPtr.Zero; this.m_curDataBlockInfo = this.m_nativeStream.ReadDataBlock(); this.m_curDataBlock = this.m_curDataBlockInfo.DataBlock; this.m_curBlockSize = this.m_curDataBlockInfo.BlockSize; this.m_curBlockPos = 0; }
internal unsafe override DataBlockInfo ReadDataBlock() { // free the dummy block if it was allocated. if (s_newlineByteBlock != null) { Marshal.FreeHGlobal((IntPtr) s_newlineByteBlock); s_newlineByteBlock = null; } while (true) { DataBlockInfo dataBlockInfo = this.m_curStream.ReadDataBlock(); if (dataBlockInfo.blockSize == 0 && this.m_curIdx == m_srcList.Count) { // data has been exhausted. We return the empty block and the caller knows what to do. return dataBlockInfo; } // normal case.. record the last two bytes for newline tracking, and return the block. if (dataBlockInfo.blockSize > 0) { if (m_appendNewLinesToFiles) { if (dataBlockInfo.blockSize >= 2) { m_trailingBytesOfData[0] = dataBlockInfo.dataBlock[dataBlockInfo.blockSize - 2]; m_trailingBytesOfData[1] = dataBlockInfo.dataBlock[dataBlockInfo.blockSize - 1]; } else { Debug.Assert(dataBlockInfo.blockSize == 1); // CASE: dataBlockInfo.blockSize == 1 // shift left. // We must do this otherwise the following data could fail to be identified // Blocks = [.........\r] [\n] m_trailingBytesOfData[0] = m_trailingBytesOfData[1]; //shift m_trailingBytesOfData[1] = dataBlockInfo.dataBlock[0]; // record the single element. } } return dataBlockInfo; } this.m_curStream.ReleaseDataBlock(dataBlockInfo.itemHandle); this.m_curStream.Close(); this.m_curStream = this.GetStream(this.m_curIdx++, FileAccess.Read); // we only get here when a file is fully consumed and it wasn't the last file in the set. // if the data stream didn't end with a newline-pair, emit one so that // LineRecord-parsing will work correctly. // the next time we enter this method, we will free the unmanaged data and the // next real block of data will be read and consumed. if (m_appendNewLinesToFiles) { //@@TODO[p3]: we currently only observe and insert \r\n pairs. // Unicode may have other types of newline to consider. if (m_trailingBytesOfData[0] != '\r' || m_trailingBytesOfData[1] != '\n') { // create the dummy block. if (s_newlineByteBlock == null) { s_newlineByteBlock = (byte*)Marshal.AllocHGlobal(2); s_newlineByteBlock[0] = (byte)'\r'; s_newlineByteBlock[1] = (byte)'\n'; } DataBlockInfo dummyblock = new DataBlockInfo(); dummyblock.blockSize = 2; dummyblock.dataBlock = s_newlineByteBlock; dummyblock.itemHandle = IntPtr.Zero; return dummyblock; } } } }
internal override unsafe DataBlockInfo ReadDataBlock() { // free the dummy block if it was allocated. if (s_newlineByteBlock != null) { Marshal.FreeHGlobal((IntPtr)s_newlineByteBlock); s_newlineByteBlock = null; } while (true) { DataBlockInfo dataBlockInfo = this.m_curStream.ReadDataBlock(); if (dataBlockInfo.blockSize == 0 && this.m_curIdx == m_srcList.Count) { // data has been exhausted. We return the empty block and the caller knows what to do. return(dataBlockInfo); } // normal case.. record the last two bytes for newline tracking, and return the block. if (dataBlockInfo.blockSize > 0) { if (m_appendNewLinesToFiles) { if (dataBlockInfo.blockSize >= 2) { m_trailingBytesOfData[0] = dataBlockInfo.dataBlock[dataBlockInfo.blockSize - 2]; m_trailingBytesOfData[1] = dataBlockInfo.dataBlock[dataBlockInfo.blockSize - 1]; } else { Debug.Assert(dataBlockInfo.blockSize == 1); // CASE: dataBlockInfo.blockSize == 1 // shift left. // We must do this otherwise the following data could fail to be identified // Blocks = [.........\r] [\n] m_trailingBytesOfData[0] = m_trailingBytesOfData[1]; //shift m_trailingBytesOfData[1] = dataBlockInfo.dataBlock[0]; // record the single element. } } return(dataBlockInfo); } this.m_curStream.ReleaseDataBlock(dataBlockInfo.itemHandle); this.m_curStream.Close(); this.m_curStream = this.GetStream(this.m_curIdx++, FileAccess.Read); // we only get here when a file is fully consumed and it wasn't the last file in the set. // if the data stream didn't end with a newline-pair, emit one so that // LineRecord-parsing will work correctly. // the next time we enter this method, we will free the unmanaged data and the // next real block of data will be read and consumed. if (m_appendNewLinesToFiles) { //@@TODO[p3]: we currently only observe and insert \r\n pairs. // Unicode may have other types of newline to consider. if (m_trailingBytesOfData[0] != '\r' || m_trailingBytesOfData[1] != '\n') { // create the dummy block. if (s_newlineByteBlock == null) { s_newlineByteBlock = (byte *)Marshal.AllocHGlobal(2); s_newlineByteBlock[0] = (byte)'\r'; s_newlineByteBlock[1] = (byte)'\n'; } DataBlockInfo dummyblock = new DataBlockInfo(); dummyblock.blockSize = 2; dummyblock.dataBlock = s_newlineByteBlock; dummyblock.itemHandle = IntPtr.Zero; return(dummyblock); } } } }
private void FlushDataBlock() { DataBlockInfo newDataBlockInfo; if (this.m_curLineStart == 0) { // The current block is too small for a single record, augment it if (this.m_curBlockSize == this.m_nextBlockSize) { throw new DryadLinqException(DryadLinqErrorCode.RecordSizeMax2GB, SR.RecordSizeMax2GB); } newDataBlockInfo = this.m_nativeStream.AllocateDataBlock(this.m_nextBlockSize); this.m_nextBlockSize = this.m_nextBlockSize * 2; if (this.m_nextBlockSize < 0) { this.m_nextBlockSize = 0x7FFFFFF8; } this.m_nativeStream.ReleaseDataBlock(this.m_curDataBlockInfo.ItemHandle); this.m_curDataBlockInfo.ItemHandle = IntPtr.Zero; } else { // Write all the complete records in the block this.m_nativeStream.WriteDataBlock(this.m_curDataBlockInfo.ItemHandle, this.m_curLineStart); this.m_numBytesWritten += this.m_curLineStart; this.m_nativeStream.ReleaseDataBlock(this.m_curDataBlockInfo.ItemHandle); this.m_curDataBlockInfo.ItemHandle = IntPtr.Zero; newDataBlockInfo = this.m_nativeStream.AllocateDataBlock(this.m_curBlockSize); this.m_curLineEnd -= this.m_curLineStart; this.m_curLineStart = 0; } this.m_curDataBlockInfo = newDataBlockInfo; this.m_curDataBlock = newDataBlockInfo.DataBlock; this.m_curBlockSize = newDataBlockInfo.BlockSize; }
/// <summary> /// Flushes the current write buffer. /// </summary> public void Flush() { Debug.Assert(this.m_curLineStart == this.m_curLineEnd); if (this.m_curLineStart > 0) { this.m_nativeStream.WriteDataBlock(this.m_curDataBlockInfo.ItemHandle, this.m_curLineStart); this.m_numBytesWritten += this.m_curLineStart; this.m_nativeStream.ReleaseDataBlock(this.m_curDataBlockInfo.ItemHandle); this.m_curDataBlockInfo.ItemHandle = IntPtr.Zero; this.m_curDataBlockInfo = this.m_nativeStream.AllocateDataBlock(this.m_curBlockSize); this.m_curDataBlock = this.m_curDataBlockInfo.DataBlock; this.m_curBlockSize = this.m_curDataBlockInfo.BlockSize; this.m_curLineStart = 0; this.m_curLineEnd = 0; } this.m_nativeStream.Flush(); }