public void WriteBytes(byte[] byteBuffer, int offset, int byteCount)
{
if (byteBuffer == null)
{
throw new ArgumentNullException("byteBuffer");
}
if (offset < 0)
{
throw new ArgumentOutOfRangeException("offset");
}
if (byteCount < 0)
{
throw new ArgumentOutOfRangeException("byteCount");
}
if (byteBuffer.Length < (offset + byteCount))
{
throw new ArgumentOutOfRangeException("byteBuffer",
String.Format(SR.ArrayLengthVsCountAndOffset,
"byteBuffer", offset + byteCount,
"offset", "byteCount"));
}
while (this.m_curBlockSize - this.m_curRecordEnd < byteCount)
{
this.FlushDataBlock();
}
fixed(byte *pBytes = byteBuffer)
{
HpcLinqUtil.memcpy(pBytes + offset, this.m_curDataBlock + this.m_curRecordEnd, byteCount);
}
this.m_curRecordEnd += byteCount;
}
/// <summary>
/// Public helper to write from a caller provided byte* to the output stream.
/// This is mainly used to read preallocated fixed size, non-integer types (Guid, decimal etc).
/// </summary>
public void WriteRawBytes(byte *pBytes, Int32 numBytes)
{
while (this.m_curBlockSize - this.m_curRecordEnd < numBytes)
{
this.FlushDataBlock();
}
HpcLinqUtil.memcpy(pBytes, this.m_curDataBlock + this.m_curRecordEnd, numBytes);
this.m_curRecordEnd += numBytes;
}
/// <summary>
/// public helper to read into a byte*, mainly used to read preallocated fixed size,
/// non-integer types (Array, Guid, decimal etc)
/// </summary>
public void ReadRawBytes(byte *pBytes, int numBytes)
{
int numBytesRead = 0;
while (numBytes > 0)
{
int numAvailBytes = this.m_curBlockSize - this.m_curBlockPos;
// if m_curDataBlock has enough bytes to fill the remainder of the user's request,
// simply copy and exit.
if (numAvailBytes >= numBytes)
{
HpcLinqUtil.memcpy(this.m_curDataBlock + this.m_curBlockPos,
pBytes + numBytesRead,
numBytes);
this.m_curBlockPos += numBytes;
break;
}
// if m_curDataBlock has less data than required, copy all the remaining bytes
// to user's buffer, update BytesRead counter, and request a new data block from
// the native stream now that m_curDataBlock is depleted
// Note that we don't need to update m_curBlockPos after memcpy() becase the
// subsequent GetNextDataBlock() call will reset it
HpcLinqUtil.memcpy(this.m_curDataBlock + this.m_curBlockPos,
pBytes + numBytesRead,
numAvailBytes);
this.GetNextDataBlock();
if (this.m_curBlockSize <= 0)
{
throw new DryadLinqException(HpcLinqErrorCode.EndOfStreamEncountered,
String.Format(SR.EndOfStreamEncountered,
GetChannelURI()));
}
numBytes -= numAvailBytes;
numBytesRead += numAvailBytes;
// here we go on to do another loop, as we can only reach when the user's request
// isn't fulfilled.
}
}
internal RangePartition(LambdaExpression keySelector,
TKey[] partitionKeys,
IComparer <TKey> comparer,
bool?isDescending,
Int32 parCnt)
: base(PartitionType.Range)
{
this.m_count = (partitionKeys == null) ? parCnt : (partitionKeys.Length + 1);
this.m_keySelector = keySelector;
this.m_partitionKeys = partitionKeys;
this.m_comparer = TypeSystem.GetComparer <TKey>(comparer);
if (isDescending == null)
{
if (partitionKeys == null)
{
throw new DryadLinqException(HpcLinqErrorCode.PartitionKeysNotProvided,
SR.PartitionKeysNotProvided);
}
bool?detectedIsDescending;
if (!HpcLinqUtil.ComputeIsDescending(partitionKeys, m_comparer, out detectedIsDescending))
{
throw new DryadLinqException(HpcLinqErrorCode.PartitionKeysAreNotConsistentlyOrdered,
SR.PartitionKeysAreNotConsistentlyOrdered);
}
this.m_isDescending = detectedIsDescending ?? false;
}
else
{
this.m_isDescending = isDescending.GetValueOrDefault();
if (partitionKeys != null &&
!HpcLinqUtil.IsOrdered(partitionKeys, this.m_comparer, this.m_isDescending))
{
throw new DryadLinqException(HpcLinqErrorCode.IsDescendingIsInconsistent,
SR.IsDescendingIsInconsistent);
}
}
}
/// <summary>
/// Private helper to write the current block out to the native stream.
/// - it writes out the current data buffer up to the point it was filled
/// - it releases the current data block back to the native stream code (which owns the lifecycle of read buffers),
/// - then allocated a new buffer from the native stream
/// - and updates the internal read buffer pointer and position members
/// </summary>
private void FlushDataBlock()
{
DataBlockInfo newDataBlockInfo;
if (this.m_curRecordStart <= 16)
{
// The current block is too small for a single record, augment it
if (this.m_curBlockSize == this.m_nextBlockSize)
{
throw new DryadLinqException(HpcLinqErrorCode.RecordSizeMax2GB, SR.RecordSizeMax2GB);
}
newDataBlockInfo = this.m_nativeStream.AllocateDataBlock(this.m_nextBlockSize);
this.m_nextBlockSize = this.m_nextBlockSize * 2;
if (this.m_nextBlockSize < 0)
{
this.m_nextBlockSize = 0x7FFFFFF8;
}
HpcLinqUtil.memcpy(this.m_curDataBlock, newDataBlockInfo.dataBlock, this.m_curRecordEnd);
}
else
{
// Write all the complete records in the block, put the partial record in the new block
newDataBlockInfo = this.m_nativeStream.AllocateDataBlock(this.m_curBlockSize);
HpcLinqUtil.memcpy(this.m_curDataBlock + this.m_curRecordStart,
newDataBlockInfo.dataBlock,
this.m_curRecordEnd - this.m_curRecordStart);
this.m_nativeStream.WriteDataBlock(this.m_curDataBlockInfo.itemHandle, this.m_curRecordStart);
this.m_numBytesWritten += this.m_curRecordStart;
this.m_curRecordEnd -= this.m_curRecordStart;
this.m_curRecordStart = 0;
}
this.m_nativeStream.ReleaseDataBlock(this.m_curDataBlockInfo.itemHandle);
this.m_curDataBlockInfo.itemHandle = IntPtr.Zero;
this.m_curDataBlockInfo = newDataBlockInfo;
this.m_curDataBlock = newDataBlockInfo.dataBlock;
this.m_curBlockSize = newDataBlockInfo.blockSize;
}
/// <summary>
/// Reads <paramref name="byteCount"/> bytes into <paramref name="destBuffer"/> starting at <paramref name="offset"/>.
/// </summary>
/// <param name="destBuffer">The pre-allocated byte array to read data into.</param>
/// <param name="offset">The starting offset at which to begin reading bytes into <paramref name="destBuffer"/>.</param>
/// <param name="byteCount">The maximum number of bytes to read. Must be smaller than or equal to (<paramref name="destBuffer.Length"/> - <paramref name="offset"/>). </param>
/// <returns>The number of bytes that was actually read.</returns>
public int ReadBytes(byte[] destBuffer, int offset, int byteCount)
{
if (destBuffer == null)
{
throw new ArgumentNullException("destBuffer");
}
if (offset < 0)
{
throw new ArgumentOutOfRangeException("offset");
}
if (byteCount < 0)
{
throw new ArgumentOutOfRangeException("byteCount");
}
if (destBuffer.Length < (offset + byteCount))
{
throw new ArgumentOutOfRangeException("destBuffer",
String.Format(SR.ArrayLengthVsCountAndOffset,
"destBuffer", offset + byteCount,
"offset", "byteCount"));
}
int numBytes = byteCount;
int numBytesRead = 0;
fixed(byte *pBytes = &destBuffer[offset])
{
while (numBytes > 0)
{
int numAvailBytes = this.m_curBlockSize - this.m_curBlockPos;
// Check if there are enough bytes in the read buffer to satisfy the
// caller's request. If so, do the copy, update m_curBlockPos and return
if (numAvailBytes >= numBytes)
{
HpcLinqUtil.memcpy(this.m_curDataBlock + this.m_curBlockPos,
pBytes + numBytesRead,
numBytes);
this.m_curBlockPos += numBytes;
numBytesRead = byteCount;
break;
}
// The remaining data in the read buffer isn't enough to fill the user's request...
// Copy the all the remaining bytes to the destination buffer, and request a
// new read buffer from the stream.
// Note that we don't need to update m_curBlockPos here because the
// GetNextDataBlock call will reset it.
HpcLinqUtil.memcpy(this.m_curDataBlock + this.m_curBlockPos,
pBytes + numBytesRead,
numAvailBytes);
// update numBytes/numBytesRead
numBytes -= numAvailBytes;
numBytesRead += numAvailBytes;
this.GetNextDataBlock();
if (this.m_curBlockSize <= 0)
{
// if the file stream returned an empty buffer it means we are at the
// end of the file. Just return the total number of bytes read, and the
// caller will decide how to handle it.
break;
}
// continue with the loop to keep filling the remaining parts of the
// destination buffer
}
}
return(numBytesRead);
}
//@@TODO[P2]: some overlap in how unique DSC stream names are made. Cleanup.
// 1. string tableUri = DryadLinqUtil.MakeUniqueName();
// string fullTableUri = DataPath.GetFullUri(tableUri);
//
// 2. string tableName = DataPath.MakeUniqueDscStreamUri();
//
// Note: both base their root path on DryadLinq.DryadOutputDir.
internal static string MakeUniqueTemporaryDscFileSetName()
{
return(TEMPORARY_STREAM_NAME_PREFIX + HpcLinqUtil.MakeUniqueName());
}