internal DryadLinqBinaryReader(NativeBlockStream stream, Encoding encoding)
{
this.m_nativeStream = stream;
this.m_encoding = encoding;
this.m_decoder = encoding.GetDecoder();
this.m_curDataBlockInfo.DataBlock = null;
this.m_curDataBlockInfo.BlockSize = -1;
this.m_curDataBlockInfo.ItemHandle = IntPtr.Zero;
this.m_curDataBlock = this.m_curDataBlockInfo.DataBlock;
this.m_curBlockSize = this.m_curDataBlockInfo.BlockSize;
this.m_curBlockPos = -1;
this.m_isClosed = false;
}
/// <summary>
/// Initializes an instance of DryadLinqTextReader.
/// </summary>
/// <param name="stream">A native stream to read from.</param>
/// <param name="encoding">The text encoding.</param>
public DryadLinqTextReader(NativeBlockStream stream, Encoding encoding)
{
this.m_nativeStream = stream;
this.m_encoding = encoding;
this.m_decoder = encoding.GetDecoder();
this.m_curDataBlockInfo.DataBlock = null;
this.m_curDataBlockInfo.BlockSize = -1;
this.m_curDataBlockInfo.ItemHandle = IntPtr.Zero;
this.m_curBlockPos = 0;
this.m_decodeUnitCharSize = this.m_encoding.GetMaxCharCount(DecodeUnitByteSize);
this.m_charBuff = new char[this.m_decodeUnitCharSize + 2]; //allow 2 bytes for trailing newline
this.m_charBuffEnd = 0;
this.m_curLineStart = 0;
this.m_curLineEnd = 0;
this.m_isClosed = false;
}
internal MultiBlockStream(List<string[]> srcList,
string associatedDscStreamName,
FileAccess access,
CompressionScheme scheme)
{
this.m_srcList = srcList;
this.m_associatedDscStreamName = associatedDscStreamName;
if (srcList.Count == 0)
{
throw new DryadLinqException(DryadLinqErrorCode.MultiBlockEmptyPartitionList,
SR.MultiBlockEmptyPartitionList);
}
this.m_compressionScheme = scheme;
this.m_curIdx = 0;
this.m_curStream = this.GetStream(this.m_curIdx++, access);
}
/// <summary>
/// Initializes an instance of DryadLinqTextWriter.
/// </summary>
/// <param name="stream">A native stream to write to.</param>
/// <param name="encoding">The text encoding.</param>
/// <param name="buffSize">A hint for the size of write buffer.</param>
public DryadLinqTextWriter(NativeBlockStream stream, Encoding encoding, Int32 buffSize)
{
this.m_nativeStream = stream;
this.m_encoding = encoding;
this.m_nextBlockSize = Math.Max(DefaultBlockSize, buffSize/2);
this.m_bufferSizeHint = buffSize;
this.m_curDataBlockInfo.DataBlock = null;
this.m_curDataBlockInfo.BlockSize = 0;
this.m_curDataBlockInfo.ItemHandle = IntPtr.Zero;
this.m_curDataBlock = this.m_curDataBlockInfo.DataBlock;
this.m_curBlockSize = this.m_curDataBlockInfo.BlockSize;
this.m_curLineStart = 0;
this.m_curLineEnd = 0;
this.m_numBytesWritten = 0;
this.m_calcFP = false;
this.m_isClosed = false;
this.m_isASCIIOrUTF8 = (encoding == Encoding.UTF8 || encoding == Encoding.ASCII);
}
internal HpcBinaryWriter(NativeBlockStream stream, Encoding encoding, Int32 buffSize)
{
this.m_nativeStream = stream;
this.m_encoding = encoding;
this.m_nextBlockSize = Math.Max(DefaultBlockSize, buffSize / 2);
this.m_bufferSizeHint = buffSize;
this.m_curDataBlockInfo.dataBlock = null;
this.m_curDataBlockInfo.blockSize = 0;
this.m_curDataBlockInfo.itemHandle = IntPtr.Zero;
this.m_curDataBlock = this.m_curDataBlockInfo.dataBlock;
this.m_curBlockSize = this.m_curDataBlockInfo.blockSize;
this.m_curRecordStart = 0;
this.m_curRecordEnd = 0;
this.m_charMaxByteCount = this.m_encoding.GetMaxByteCount(1);
this.m_isClosed = false;
this.m_numBytesWritten = 0;
this.m_calcFP = false;
this.m_bfm = new BinaryFormatter();
}
public override HpcRecordReader <ushort> MakeReader(NativeBlockStream nativeStream)
{
return(new HpcRecordUShortReader(new HpcBinaryReader(nativeStream)));
}
public override HpcRecordReader <int> MakeReader(NativeBlockStream nativeStream)
{
return(new HpcRecordInt32Reader(new HpcBinaryReader(nativeStream)));
}
/// <summary>
/// Initializes an instance of DryadLinqTextWriter with encoding UTF8.
/// </summary>
/// <param name="stream">A native stream to write to.</param>
public DryadLinqTextWriter(NativeBlockStream stream)
: this(stream, Encoding.UTF8)
{
}
public override HpcRecordReader <char> MakeReader(NativeBlockStream nativeStream)
{
return(new HpcRecordCharReader(new HpcBinaryReader(nativeStream)));
}
public override DryadLinqRecordReader <decimal> MakeReader(NativeBlockStream nativeStream)
{
return(new DryadLinqRecordDecimalReader(new DryadLinqBinaryReader(nativeStream)));
}
public override DryadLinqRecordWriter <LineRecord> MakeWriter(NativeBlockStream nativeStream)
{
return(new DryadLinqRecordTextWriter(new DryadLinqTextWriter(nativeStream)));
}
public abstract void Write(NativeBlockStream stream);
public override DryadLinqRecordReader <float> MakeReader(NativeBlockStream nativeStream)
{
return(new DryadLinqRecordFloatReader(new DryadLinqBinaryReader(nativeStream)));
}
public override DryadLinqRecordReader <sbyte> MakeReader(NativeBlockStream nativeStream)
{
return(new DryadLinqRecordSByteReader(new DryadLinqBinaryReader(nativeStream)));
}
public static HpcBinaryWriter MakeBinaryWriter(NativeBlockStream nativeStream)
{
return(new HpcBinaryWriter(nativeStream));
}
public override DryadLinqRecordWriter <byte> MakeWriter(NativeBlockStream nativeStream)
{
return(new DryadLinqRecordByteWriter(new DryadLinqBinaryWriter(nativeStream)));
}
/// <summary> /// Creates a new writer from a native stream. /// </summary> /// <param name="nativeStream">A native stream.</param> /// <returns>A writer that uses DryadLINQ serialization for record of type T.</returns> public abstract DryadLinqRecordWriter <T> MakeWriter(NativeBlockStream nativeStream);
public override DryadLinqRecordWriter <SqlDateTime> MakeWriter(NativeBlockStream nativeStream)
{
return(new DryadLinqRecordSqlDateTimeWriter(new DryadLinqBinaryWriter(nativeStream)));
}
public override HpcRecordReader <ulong> MakeReader(NativeBlockStream nativeStream)
{
return(new HpcRecordUInt64Reader(new HpcBinaryReader(nativeStream)));
}
/// <summary>
/// Initializes an instance of DryadLinqTextWriter.
/// </summary>
/// <param name="stream">A native stream to write to.</param>
/// <param name="encoding">The text encoding.</param>
public DryadLinqTextWriter(NativeBlockStream stream, Encoding encoding)
: this(stream, encoding, DefaultBlockSize)
{
}
public override DryadLinqRecordReader <DateTime> MakeReader(NativeBlockStream nativeStream)
{
return(new DryadLinqRecordDateTimeReader(new DryadLinqBinaryReader(nativeStream)));
}
/// <summary>
/// Make a binary writer from a native stream. Used only by auto-generated code.
/// </summary>
/// <param name="nativeStream">A native stream</param>
/// <returns>A binary writer</returns>
public static DryadLinqBinaryWriter MakeBinaryWriter(NativeBlockStream nativeStream)
{
return(new DryadLinqBinaryWriter(nativeStream));
}
internal DryadLinqBinaryReader(NativeBlockStream stream)
: this(stream, Encoding.UTF8)
{
}
internal HpcBinaryWriter(NativeBlockStream stream, Encoding encoding)
: this(stream, encoding, DefaultBlockSize)
{
}
public HpcTextReader(NativeBlockStream stream)
: this(stream, Encoding.UTF8)
{
}
/// <summary> /// Creates a new reader from a native stream. /// </summary> /// <param name="nativeStream">A native stream.</param> /// <returns>A reader that uses DryadLINQ serialization for records of type T.</returns> public abstract DryadLinqRecordReader <T> MakeReader(NativeBlockStream nativeStream);
internal HpcBinaryWriter(NativeBlockStream stream)
: this(stream, Encoding.UTF8)
{
}
internal override unsafe DataBlockInfo ReadDataBlock()
{
// free the dummy block if it was allocated.
if (s_newlineByteBlock != null)
{
Marshal.FreeHGlobal((IntPtr)s_newlineByteBlock);
s_newlineByteBlock = null;
}
while (true)
{
DataBlockInfo dataBlockInfo = this.m_curStream.ReadDataBlock();
if (dataBlockInfo.blockSize == 0 && this.m_curIdx == m_srcList.Count)
{
// data has been exhausted. We return the empty block and the caller knows what to do.
return(dataBlockInfo);
}
// normal case.. record the last two bytes for newline tracking, and return the block.
if (dataBlockInfo.blockSize > 0)
{
if (m_appendNewLinesToFiles)
{
if (dataBlockInfo.blockSize >= 2)
{
m_trailingBytesOfData[0] = dataBlockInfo.dataBlock[dataBlockInfo.blockSize - 2];
m_trailingBytesOfData[1] = dataBlockInfo.dataBlock[dataBlockInfo.blockSize - 1];
}
else
{
Debug.Assert(dataBlockInfo.blockSize == 1);
// CASE: dataBlockInfo.blockSize == 1
// shift left.
// We must do this otherwise the following data could fail to be identified
// Blocks = [.........\r] [\n]
m_trailingBytesOfData[0] = m_trailingBytesOfData[1]; //shift
m_trailingBytesOfData[1] = dataBlockInfo.dataBlock[0]; // record the single element.
}
}
return(dataBlockInfo);
}
this.m_curStream.ReleaseDataBlock(dataBlockInfo.itemHandle);
this.m_curStream.Close();
this.m_curStream = this.GetStream(this.m_curIdx++, FileAccess.Read);
// we only get here when a file is fully consumed and it wasn't the last file in the set.
// if the data stream didn't end with a newline-pair, emit one so that
// LineRecord-parsing will work correctly.
// the next time we enter this method, we will free the unmanaged data and the
// next real block of data will be read and consumed.
if (m_appendNewLinesToFiles)
{
//@@TODO[p3]: we currently only observe and insert \r\n pairs.
// Unicode may have other types of newline to consider.
if (m_trailingBytesOfData[0] != '\r' || m_trailingBytesOfData[1] != '\n')
{
// create the dummy block.
if (s_newlineByteBlock == null)
{
s_newlineByteBlock = (byte *)Marshal.AllocHGlobal(2);
s_newlineByteBlock[0] = (byte)'\r';
s_newlineByteBlock[1] = (byte)'\n';
}
DataBlockInfo dummyblock = new DataBlockInfo();
dummyblock.blockSize = 2;
dummyblock.dataBlock = s_newlineByteBlock;
dummyblock.itemHandle = IntPtr.Zero;
return(dummyblock);
}
}
}
}
public abstract void Write(NativeBlockStream stream);
private NativeBlockStream m_nativeStream; // source stream
#endregion Fields
#region Constructors
public HpcTextReader(NativeBlockStream stream)
: this(stream, Encoding.UTF8)
{
}
/// <summary>
/// Initializes an instance of DryadLinqTextWriter.
/// </summary>
/// <param name="stream">A native stream to write to.</param>
/// <param name="encoding">The text encoding.</param>
public DryadLinqTextWriter(NativeBlockStream stream, Encoding encoding)
: this(stream, encoding, DefaultBlockSize)
{
}
internal unsafe override DataBlockInfo ReadDataBlock()
{
// free the dummy block if it was allocated.
if (s_newlineByteBlock != null)
{
Marshal.FreeHGlobal((IntPtr) s_newlineByteBlock);
s_newlineByteBlock = null;
}
while (true)
{
DataBlockInfo dataBlockInfo = this.m_curStream.ReadDataBlock();
if (dataBlockInfo.blockSize == 0 && this.m_curIdx == m_srcList.Count)
{
// data has been exhausted. We return the empty block and the caller knows what to do.
return dataBlockInfo;
}
// normal case.. record the last two bytes for newline tracking, and return the block.
if (dataBlockInfo.blockSize > 0)
{
if (m_appendNewLinesToFiles)
{
if (dataBlockInfo.blockSize >= 2)
{
m_trailingBytesOfData[0] = dataBlockInfo.dataBlock[dataBlockInfo.blockSize - 2];
m_trailingBytesOfData[1] = dataBlockInfo.dataBlock[dataBlockInfo.blockSize - 1];
}
else
{
Debug.Assert(dataBlockInfo.blockSize == 1);
// CASE: dataBlockInfo.blockSize == 1
// shift left.
// We must do this otherwise the following data could fail to be identified
// Blocks = [.........\r] [\n]
m_trailingBytesOfData[0] = m_trailingBytesOfData[1]; //shift
m_trailingBytesOfData[1] = dataBlockInfo.dataBlock[0]; // record the single element.
}
}
return dataBlockInfo;
}
this.m_curStream.ReleaseDataBlock(dataBlockInfo.itemHandle);
this.m_curStream.Close();
this.m_curStream = this.GetStream(this.m_curIdx++, FileAccess.Read);
// we only get here when a file is fully consumed and it wasn't the last file in the set.
// if the data stream didn't end with a newline-pair, emit one so that
// LineRecord-parsing will work correctly.
// the next time we enter this method, we will free the unmanaged data and the
// next real block of data will be read and consumed.
if (m_appendNewLinesToFiles)
{
//@@TODO[p3]: we currently only observe and insert \r\n pairs.
// Unicode may have other types of newline to consider.
if (m_trailingBytesOfData[0] != '\r' || m_trailingBytesOfData[1] != '\n')
{
// create the dummy block.
if (s_newlineByteBlock == null)
{
s_newlineByteBlock = (byte*)Marshal.AllocHGlobal(2);
s_newlineByteBlock[0] = (byte)'\r';
s_newlineByteBlock[1] = (byte)'\n';
}
DataBlockInfo dummyblock = new DataBlockInfo();
dummyblock.blockSize = 2;
dummyblock.dataBlock = s_newlineByteBlock;
dummyblock.itemHandle = IntPtr.Zero;
return dummyblock;
}
}
}
}
public override HpcRecordWriter <char> MakeWriter(NativeBlockStream nativeStream)
{
return(new HpcRecordCharWriter(new HpcBinaryWriter(nativeStream)));
}
/// <summary>
/// Initializes an instance of DryadLinqTextReader with encoding UTF8.
/// </summary>
/// <param name="stream">A native stream to read from.</param>
public DryadLinqTextReader(NativeBlockStream stream)
: this(stream, Encoding.UTF8)
{
}
public override HpcRecordWriter <ushort> MakeWriter(NativeBlockStream nativeStream)
{
return(new HpcRecordUShortWriter(new HpcBinaryWriter(nativeStream)));
}
public override HpcRecordReader <float> MakeReader(NativeBlockStream nativeStream)
{
return(new HpcRecordFloatReader(new HpcBinaryReader(nativeStream)));
}
public override HpcRecordWriter <int> MakeWriter(NativeBlockStream nativeStream)
{
return(new HpcRecordInt32Writer(new HpcBinaryWriter(nativeStream)));
}
public override HpcRecordReader <decimal> MakeReader(NativeBlockStream nativeStream)
{
return(new HpcRecordDecimalReader(new HpcBinaryReader(nativeStream)));
}
public override HpcRecordWriter <ulong> MakeWriter(NativeBlockStream nativeStream)
{
return(new HpcRecordUInt64Writer(new HpcBinaryWriter(nativeStream)));
}
public override DryadLinqRecordWriter <decimal> MakeWriter(NativeBlockStream nativeStream)
{
return(new DryadLinqRecordDecimalWriter(new DryadLinqBinaryWriter(nativeStream)));
}
public override HpcRecordWriter <float> MakeWriter(NativeBlockStream nativeStream)
{
return(new HpcRecordFloatWriter(new HpcBinaryWriter(nativeStream)));
}
internal override unsafe DataBlockInfo ReadDataBlock()
{
while (true)
{
DataBlockInfo dataBlockInfo = this.m_curStream.ReadDataBlock();
if (dataBlockInfo.BlockSize == 0 && this.m_curIdx == m_srcList.Count)
{
// data has been exhausted. We return the empty block and the caller knows what to do.
return dataBlockInfo;
}
// normal case.. record the last two bytes for newline tracking, and return the block.
if (dataBlockInfo.BlockSize > 0)
{
return dataBlockInfo;
}
this.m_curStream.ReleaseDataBlock(dataBlockInfo.ItemHandle);
this.m_curStream.Close();
this.m_curStream = this.GetStream(this.m_curIdx++, FileAccess.Read);
}
}
public override HpcRecordWriter <decimal> MakeWriter(NativeBlockStream nativeStream)
{
return(new HpcRecordDecimalWriter(new HpcBinaryWriter(nativeStream)));
}
public override DryadLinqRecordWriter <ulong> MakeWriter(NativeBlockStream nativeStream)
{
return(new DryadLinqRecordUInt64Writer(new DryadLinqBinaryWriter(nativeStream)));
}