private void FlushDataBlock()
{
DataBlockInfo newDataBlockInfo;
if (this.m_curLineStart == 0)
{
// The current block is too small for a single record, augment it
if (this.m_curBlockSize == this.m_nextBlockSize)
{
throw new DryadLinqException(DryadLinqErrorCode.RecordSizeMax2GB, SR.RecordSizeMax2GB);
}
newDataBlockInfo = this.m_nativeStream.AllocateDataBlock(this.m_nextBlockSize);
this.m_nextBlockSize = this.m_nextBlockSize * 2;
if (this.m_nextBlockSize < 0)
{
this.m_nextBlockSize = 0x7FFFFFF8;
}
this.m_nativeStream.ReleaseDataBlock(this.m_curDataBlockInfo.ItemHandle);
this.m_curDataBlockInfo.ItemHandle = IntPtr.Zero;
}
else
{
// Write all the complete records in the block
this.m_nativeStream.WriteDataBlock(this.m_curDataBlockInfo.ItemHandle, this.m_curLineStart);
this.m_numBytesWritten += this.m_curLineStart;
this.m_nativeStream.ReleaseDataBlock(this.m_curDataBlockInfo.ItemHandle);
this.m_curDataBlockInfo.ItemHandle = IntPtr.Zero;
newDataBlockInfo = this.m_nativeStream.AllocateDataBlock(this.m_curBlockSize);
this.m_curLineEnd -= this.m_curLineStart;
this.m_curLineStart = 0;
}
this.m_curDataBlockInfo = newDataBlockInfo;
this.m_curDataBlock = newDataBlockInfo.DataBlock;
this.m_curBlockSize = newDataBlockInfo.BlockSize;
}
// Get the next data block
private unsafe void GetNextDataBlock()
{
this.m_nativeStream.ReleaseDataBlock(this.m_curDataBlockInfo.ItemHandle);
this.m_curDataBlockInfo.ItemHandle = IntPtr.Zero;
this.m_curDataBlockInfo = this.m_nativeStream.ReadDataBlock();
this.m_curBlockPos = 0;
}
/// <summary>
/// Flushes the current write buffer.
/// </summary>
public void Flush()
{
Debug.Assert(this.m_curLineStart == this.m_curLineEnd);
if (this.m_curLineStart > 0)
{
this.m_nativeStream.WriteDataBlock(this.m_curDataBlockInfo.ItemHandle, this.m_curLineStart);
this.m_numBytesWritten += this.m_curLineStart;
this.m_nativeStream.ReleaseDataBlock(this.m_curDataBlockInfo.ItemHandle);
this.m_curDataBlockInfo.ItemHandle = IntPtr.Zero;
this.m_curDataBlockInfo = this.m_nativeStream.AllocateDataBlock(this.m_curBlockSize);
this.m_curDataBlock = this.m_curDataBlockInfo.DataBlock;
this.m_curBlockSize = this.m_curDataBlockInfo.BlockSize;
this.m_curLineStart = 0;
this.m_curLineEnd = 0;
}
this.m_nativeStream.Flush();
}
internal override unsafe DataBlockInfo ReadDataBlock()
{
while (true)
{
DataBlockInfo dataBlockInfo = this.m_curStream.ReadDataBlock();
if (dataBlockInfo.BlockSize == 0 && this.m_curIdx == m_srcList.Count)
{
// data has been exhausted. We return the empty block and the caller knows what to do.
return(dataBlockInfo);
}
// normal case.. record the last two bytes for newline tracking, and return the block.
if (dataBlockInfo.BlockSize > 0)
{
return(dataBlockInfo);
}
this.m_curStream.ReleaseDataBlock(dataBlockInfo.ItemHandle);
this.m_curStream.Close();
this.m_curStream = this.GetStream(this.m_curIdx++, FileAccess.Read);
}
}
/// <summary>
/// Private helper to write the current block out to the native stream.
/// - it writes out the current data buffer up to the point it was filled
/// - it releases the current data block back to the native stream code (which owns the lifecycle of read buffers),
/// - then allocated a new buffer from the native stream
/// - and updates the internal read buffer pointer and position members
/// </summary>
private void FlushDataBlock()
{
DataBlockInfo newDataBlockInfo;
if (this.m_curRecordStart <= 16)
{
// The current block is too small for a single record, augment it
if (this.m_curBlockSize == this.m_nextBlockSize)
{
throw new DryadLinqException(HpcLinqErrorCode.RecordSizeMax2GB, SR.RecordSizeMax2GB);
}
newDataBlockInfo = this.m_nativeStream.AllocateDataBlock(this.m_nextBlockSize);
this.m_nextBlockSize = this.m_nextBlockSize * 2;
if (this.m_nextBlockSize < 0)
{
this.m_nextBlockSize = 0x7FFFFFF8;
}
HpcLinqUtil.memcpy(this.m_curDataBlock, newDataBlockInfo.dataBlock, this.m_curRecordEnd);
}
else
{
// Write all the complete records in the block, put the partial record in the new block
newDataBlockInfo = this.m_nativeStream.AllocateDataBlock(this.m_curBlockSize);
HpcLinqUtil.memcpy(this.m_curDataBlock + this.m_curRecordStart,
newDataBlockInfo.dataBlock,
this.m_curRecordEnd - this.m_curRecordStart);
this.m_nativeStream.WriteDataBlock(this.m_curDataBlockInfo.itemHandle, this.m_curRecordStart);
this.m_numBytesWritten += this.m_curRecordStart;
this.m_curRecordEnd -= this.m_curRecordStart;
this.m_curRecordStart = 0;
}
this.m_nativeStream.ReleaseDataBlock(this.m_curDataBlockInfo.itemHandle);
this.m_curDataBlockInfo.itemHandle = IntPtr.Zero;
this.m_curDataBlockInfo = newDataBlockInfo;
this.m_curDataBlock = newDataBlockInfo.dataBlock;
this.m_curBlockSize = newDataBlockInfo.blockSize;
}
/// <summary>
/// Writes out the current data buffer (equivalent of FlushDataBlock), and calls
/// Flush on the native stream to ensure all the data makes its way to the disk
/// </summary>
internal void Flush()
{
if (this.m_curRecordEnd > 0)
{
this.m_nativeStream.WriteDataBlock(this.m_curDataBlockInfo.itemHandle, this.m_curRecordEnd);
this.m_numBytesWritten += this.m_curRecordEnd;
this.m_nativeStream.ReleaseDataBlock(this.m_curDataBlockInfo.itemHandle);
this.m_curDataBlockInfo.itemHandle = IntPtr.Zero;
this.m_curDataBlockInfo = this.m_nativeStream.AllocateDataBlock(this.m_curBlockSize);
this.m_curDataBlock = this.m_curDataBlockInfo.dataBlock;
this.m_curBlockSize = this.m_curDataBlockInfo.blockSize;
this.m_curRecordStart = 0;
this.m_curRecordEnd = 0;
}
this.m_nativeStream.Flush();
}
/// <summary>
/// Private helper to request a new data block from the native stream.
/// - it releases the current data block back to the native stream code
/// (which owns the lifecycle of read buffers),
/// - then requests a new buffer from the native stream
/// - and updates the internal read buffer pointer (m_curDataBlock), size (m_curDataSize)
/// and position, all of which are needed by subsequent Read*() calls coming from the user
/// </summary>
private unsafe void GetNextDataBlock()
{
this.m_nativeStream.ReleaseDataBlock(this.m_curDataBlockInfo.ItemHandle);
this.m_curDataBlockInfo.ItemHandle = IntPtr.Zero;
this.m_curDataBlockInfo = this.m_nativeStream.ReadDataBlock();
this.m_curDataBlock = this.m_curDataBlockInfo.DataBlock;
this.m_curBlockSize = this.m_curDataBlockInfo.BlockSize;
this.m_curBlockPos = 0;
}
internal unsafe override DataBlockInfo ReadDataBlock()
{
// free the dummy block if it was allocated.
if (s_newlineByteBlock != null)
{
Marshal.FreeHGlobal((IntPtr) s_newlineByteBlock);
s_newlineByteBlock = null;
}
while (true)
{
DataBlockInfo dataBlockInfo = this.m_curStream.ReadDataBlock();
if (dataBlockInfo.blockSize == 0 && this.m_curIdx == m_srcList.Count)
{
// data has been exhausted. We return the empty block and the caller knows what to do.
return dataBlockInfo;
}
// normal case.. record the last two bytes for newline tracking, and return the block.
if (dataBlockInfo.blockSize > 0)
{
if (m_appendNewLinesToFiles)
{
if (dataBlockInfo.blockSize >= 2)
{
m_trailingBytesOfData[0] = dataBlockInfo.dataBlock[dataBlockInfo.blockSize - 2];
m_trailingBytesOfData[1] = dataBlockInfo.dataBlock[dataBlockInfo.blockSize - 1];
}
else
{
Debug.Assert(dataBlockInfo.blockSize == 1);
// CASE: dataBlockInfo.blockSize == 1
// shift left.
// We must do this otherwise the following data could fail to be identified
// Blocks = [.........\r] [\n]
m_trailingBytesOfData[0] = m_trailingBytesOfData[1]; //shift
m_trailingBytesOfData[1] = dataBlockInfo.dataBlock[0]; // record the single element.
}
}
return dataBlockInfo;
}
this.m_curStream.ReleaseDataBlock(dataBlockInfo.itemHandle);
this.m_curStream.Close();
this.m_curStream = this.GetStream(this.m_curIdx++, FileAccess.Read);
// we only get here when a file is fully consumed and it wasn't the last file in the set.
// if the data stream didn't end with a newline-pair, emit one so that
// LineRecord-parsing will work correctly.
// the next time we enter this method, we will free the unmanaged data and the
// next real block of data will be read and consumed.
if (m_appendNewLinesToFiles)
{
//@@TODO[p3]: we currently only observe and insert \r\n pairs.
// Unicode may have other types of newline to consider.
if (m_trailingBytesOfData[0] != '\r' || m_trailingBytesOfData[1] != '\n')
{
// create the dummy block.
if (s_newlineByteBlock == null)
{
s_newlineByteBlock = (byte*)Marshal.AllocHGlobal(2);
s_newlineByteBlock[0] = (byte)'\r';
s_newlineByteBlock[1] = (byte)'\n';
}
DataBlockInfo dummyblock = new DataBlockInfo();
dummyblock.blockSize = 2;
dummyblock.dataBlock = s_newlineByteBlock;
dummyblock.itemHandle = IntPtr.Zero;
return dummyblock;
}
}
}
}
internal override unsafe DataBlockInfo ReadDataBlock()
{
// free the dummy block if it was allocated.
if (s_newlineByteBlock != null)
{
Marshal.FreeHGlobal((IntPtr)s_newlineByteBlock);
s_newlineByteBlock = null;
}
while (true)
{
DataBlockInfo dataBlockInfo = this.m_curStream.ReadDataBlock();
if (dataBlockInfo.blockSize == 0 && this.m_curIdx == m_srcList.Count)
{
// data has been exhausted. We return the empty block and the caller knows what to do.
return(dataBlockInfo);
}
// normal case.. record the last two bytes for newline tracking, and return the block.
if (dataBlockInfo.blockSize > 0)
{
if (m_appendNewLinesToFiles)
{
if (dataBlockInfo.blockSize >= 2)
{
m_trailingBytesOfData[0] = dataBlockInfo.dataBlock[dataBlockInfo.blockSize - 2];
m_trailingBytesOfData[1] = dataBlockInfo.dataBlock[dataBlockInfo.blockSize - 1];
}
else
{
Debug.Assert(dataBlockInfo.blockSize == 1);
// CASE: dataBlockInfo.blockSize == 1
// shift left.
// We must do this otherwise the following data could fail to be identified
// Blocks = [.........\r] [\n]
m_trailingBytesOfData[0] = m_trailingBytesOfData[1]; //shift
m_trailingBytesOfData[1] = dataBlockInfo.dataBlock[0]; // record the single element.
}
}
return(dataBlockInfo);
}
this.m_curStream.ReleaseDataBlock(dataBlockInfo.itemHandle);
this.m_curStream.Close();
this.m_curStream = this.GetStream(this.m_curIdx++, FileAccess.Read);
// we only get here when a file is fully consumed and it wasn't the last file in the set.
// if the data stream didn't end with a newline-pair, emit one so that
// LineRecord-parsing will work correctly.
// the next time we enter this method, we will free the unmanaged data and the
// next real block of data will be read and consumed.
if (m_appendNewLinesToFiles)
{
//@@TODO[p3]: we currently only observe and insert \r\n pairs.
// Unicode may have other types of newline to consider.
if (m_trailingBytesOfData[0] != '\r' || m_trailingBytesOfData[1] != '\n')
{
// create the dummy block.
if (s_newlineByteBlock == null)
{
s_newlineByteBlock = (byte *)Marshal.AllocHGlobal(2);
s_newlineByteBlock[0] = (byte)'\r';
s_newlineByteBlock[1] = (byte)'\n';
}
DataBlockInfo dummyblock = new DataBlockInfo();
dummyblock.blockSize = 2;
dummyblock.dataBlock = s_newlineByteBlock;
dummyblock.itemHandle = IntPtr.Zero;
return(dummyblock);
}
}
}
}
private void FlushDataBlock()
{
DataBlockInfo newDataBlockInfo;
if (this.m_curLineStart == 0)
{
// The current block is too small for a single record, augment it
if (this.m_curBlockSize == this.m_nextBlockSize)
{
throw new DryadLinqException(DryadLinqErrorCode.RecordSizeMax2GB, SR.RecordSizeMax2GB);
}
newDataBlockInfo = this.m_nativeStream.AllocateDataBlock(this.m_nextBlockSize);
this.m_nextBlockSize = this.m_nextBlockSize * 2;
if (this.m_nextBlockSize < 0)
{
this.m_nextBlockSize = 0x7FFFFFF8;
}
this.m_nativeStream.ReleaseDataBlock(this.m_curDataBlockInfo.ItemHandle);
this.m_curDataBlockInfo.ItemHandle = IntPtr.Zero;
}
else
{
// Write all the complete records in the block
this.m_nativeStream.WriteDataBlock(this.m_curDataBlockInfo.ItemHandle, this.m_curLineStart);
this.m_numBytesWritten += this.m_curLineStart;
this.m_nativeStream.ReleaseDataBlock(this.m_curDataBlockInfo.ItemHandle);
this.m_curDataBlockInfo.ItemHandle = IntPtr.Zero;
newDataBlockInfo = this.m_nativeStream.AllocateDataBlock(this.m_curBlockSize);
this.m_curLineEnd -= this.m_curLineStart;
this.m_curLineStart = 0;
}
this.m_curDataBlockInfo = newDataBlockInfo;
this.m_curDataBlock = newDataBlockInfo.DataBlock;
this.m_curBlockSize = newDataBlockInfo.BlockSize;
}
/// <summary>
/// Flushes the current write buffer.
/// </summary>
public void Flush()
{
Debug.Assert(this.m_curLineStart == this.m_curLineEnd);
if (this.m_curLineStart > 0)
{
this.m_nativeStream.WriteDataBlock(this.m_curDataBlockInfo.ItemHandle, this.m_curLineStart);
this.m_numBytesWritten += this.m_curLineStart;
this.m_nativeStream.ReleaseDataBlock(this.m_curDataBlockInfo.ItemHandle);
this.m_curDataBlockInfo.ItemHandle = IntPtr.Zero;
this.m_curDataBlockInfo = this.m_nativeStream.AllocateDataBlock(this.m_curBlockSize);
this.m_curDataBlock = this.m_curDataBlockInfo.DataBlock;
this.m_curBlockSize = this.m_curDataBlockInfo.BlockSize;
this.m_curLineStart = 0;
this.m_curLineEnd = 0;
}
this.m_nativeStream.Flush();
}
