/// <summary>
/// Writes an array of bytes to the current writer.
/// </summary>
/// <param name="byteBuffer">The byte array to write.</param>
/// <param name="offset">The starting index.</param>
/// <param name="byteCount">The number of bytes to write.</param>
public void WriteBytes(byte[] byteBuffer, int offset, int byteCount)
{
if (byteBuffer == null)
{
throw new ArgumentNullException("byteBuffer");
}
if (offset < 0)
{
throw new ArgumentOutOfRangeException("offset");
}
if (byteCount < 0)
{
throw new ArgumentOutOfRangeException("byteCount");
}
if (byteBuffer.Length < (offset + byteCount))
{
throw new ArgumentOutOfRangeException("byteBuffer",
String.Format(SR.ArrayLengthVsCountAndOffset,
"byteBuffer", offset + byteCount,
"offset", "byteCount"));
}
while (this.m_curBlockSize - this.m_curRecordEnd < byteCount)
{
this.FlushDataBlock();
}
fixed(byte *pBytes = byteBuffer)
{
DryadLinqUtil.memcpy(pBytes + offset, this.m_curDataBlock + this.m_curRecordEnd, byteCount);
}
this.m_curRecordEnd += byteCount;
}
/// <summary>
/// Public helper to write from a caller provided byte* to the output stream.
/// This is mainly used to read preallocated fixed size, non-integer types (Guid, decimal etc).
/// </summary>
/// <param name="pBytes">A pointer to the byte array to write</param>
/// <param name="numBytes">The number of bytes to write</param>
public void WriteRawBytes(byte *pBytes, Int32 numBytes)
{
while (this.m_curBlockSize - this.m_curRecordEnd < numBytes)
{
this.FlushDataBlock();
}
DryadLinqUtil.memcpy(pBytes, this.m_curDataBlock + this.m_curRecordEnd, numBytes);
this.m_curRecordEnd += numBytes;
}
internal Uri MakeTemporaryStreamUri()
{
if (this._storageSetScheme == null)
{
throw new DryadLinqException("The storage scheme for temporary streams must be specified.");
}
DataProvider dataProvider = DataProvider.GetDataProvider(this._storageSetScheme);
return(dataProvider.GetTemporaryStreamUri(this, DryadLinqUtil.MakeUniqueName()));
}
public override void Ingress <T>(DryadLinqContext context,
IEnumerable <T> source,
Uri dataSetUri,
DryadLinqMetaData metaData,
CompressionScheme compressionScheme,
bool isTemp,
Expression <Action <IEnumerable <T>, Stream> > serializer)
{
string fileName = dataSetUri.LocalPath;
if (!String.IsNullOrEmpty(dataSetUri.Host))
{
fileName = @"\\" + dataSetUri.Host + fileName;
}
// Write the partition:
string partDir = Path.GetDirectoryName(fileName);
partDir = Path.Combine(partDir, DryadLinqUtil.MakeUniqueName());
Directory.CreateDirectory(partDir);
string uncPath = Path.Combine(partDir, "Part");
string partitionPath = uncPath + ".00000000";
DryadLinqFactory <T> factory = (DryadLinqFactory <T>)DryadLinqCodeGen.GetFactory(context, typeof(T));
using (FileStream fstream = new FileStream(partitionPath, FileMode.CreateNew, FileAccess.Write))
{
if (serializer == null)
{
DryadLinqFileBlockStream nativeStream = new DryadLinqFileBlockStream(fstream, compressionScheme);
DryadLinqRecordWriter <T> writer = factory.MakeWriter(nativeStream);
foreach (T rec in source)
{
writer.WriteRecordSync(rec);
}
writer.Close();
}
else
{
Action <IEnumerable <T>, Stream> serializerFunc = serializer.Compile();
serializerFunc(source, fstream);
}
}
// Write the partfile:
long partSize = new FileInfo(partitionPath).Length;
using (StreamWriter writer = File.CreateText(fileName))
{
writer.WriteLine(uncPath);
writer.WriteLine("1");
writer.WriteLine("{0},{1}", 0, partSize);
}
}
public override void Ingress <T>(DryadLinqContext context,
IEnumerable <T> source,
Uri dataSetUri,
DryadLinqMetaData metaData,
CompressionScheme compressionScheme,
bool isTemp = false)
{
// Write the partition:
string partDir = context.PartitionUncPath;
if (partDir == null)
{
partDir = Path.GetDirectoryName(dataSetUri.LocalPath);
}
if (!Path.IsPathRooted(partDir))
{
partDir = Path.Combine("/", partDir);
}
partDir = Path.Combine(partDir, DryadLinqUtil.MakeUniqueName());
Directory.CreateDirectory(partDir);
string partPath = Path.Combine(partDir, "Part");
string partFilePath = partPath + ".00000000";
DryadLinqFactory <T> factory = (DryadLinqFactory <T>)DryadLinqCodeGen.GetFactory(context, typeof(T));
using (FileStream fstream = new FileStream(partFilePath, FileMode.CreateNew, FileAccess.Write))
{
DryadLinqFileBlockStream nativeStream = new DryadLinqFileBlockStream(fstream, compressionScheme);
DryadLinqRecordWriter <T> writer = factory.MakeWriter(nativeStream);
foreach (T rec in source)
{
writer.WriteRecordSync(rec);
}
writer.Close();
}
// Write the partfile:
FileInfo finfo = new FileInfo(partFilePath);
using (StreamWriter writer = File.CreateText(dataSetUri.LocalPath))
{
writer.WriteLine(partPath);
writer.WriteLine("1");
writer.WriteLine("{0},{1},{2}", 0, finfo.Length, Environment.MachineName);
}
}
/// <summary>
/// public helper to read into a byte*, mainly used to read preallocated fixed size,
/// non-integer types (Array, Guid, decimal etc)
/// </summary>
/// <param name="pBytes">The pointer to the pre-allocated byte array to read data into</param>
/// <param name="numBytes">The number of bytes to read</param>
public void ReadRawBytes(byte *pBytes, int numBytes)
{
int numBytesRead = 0;
while (numBytes > 0)
{
int numAvailBytes = this.m_curBlockSize - this.m_curBlockPos;
// if m_curDataBlock has enough bytes to fill the remainder of the user's request,
// simply copy and exit.
if (numAvailBytes >= numBytes)
{
DryadLinqUtil.memcpy(this.m_curDataBlock + this.m_curBlockPos,
pBytes + numBytesRead,
numBytes);
this.m_curBlockPos += numBytes;
break;
}
// if m_curDataBlock has less data than required, copy all the remaining bytes
// to user's buffer, update BytesRead counter, and request a new data block from
// the native stream now that m_curDataBlock is depleted
// Note that we don't need to update m_curBlockPos after memcpy() becase the
// subsequent GetNextDataBlock() call will reset it
DryadLinqUtil.memcpy(this.m_curDataBlock + this.m_curBlockPos,
pBytes + numBytesRead,
numAvailBytes);
this.GetNextDataBlock();
if (this.m_curBlockSize <= 0)
{
throw new DryadLinqException(DryadLinqErrorCode.EndOfStreamEncountered,
String.Format(SR.EndOfStreamEncountered,
GetChannelURI()));
}
numBytes -= numAvailBytes;
numBytesRead += numAvailBytes;
// here we go on to do another loop, as we can only reach when the user's request
// isn't fulfilled.
}
}
internal RangePartition(LambdaExpression keySelector,
TKey[] partitionKeys,
IComparer <TKey> comparer,
bool?isDescending,
Int32 parCnt)
: base(PartitionType.Range)
{
this.m_count = (partitionKeys == null) ? parCnt : (partitionKeys.Length + 1);
this.m_keySelector = keySelector;
this.m_partitionKeys = partitionKeys;
this.m_comparer = TypeSystem.GetComparer <TKey>(comparer);
if (isDescending == null)
{
if (partitionKeys == null)
{
throw new DryadLinqException(DryadLinqErrorCode.PartitionKeysNotProvided,
SR.PartitionKeysNotProvided);
}
bool?detectedIsDescending;
if (!DryadLinqUtil.ComputeIsDescending(partitionKeys, m_comparer, out detectedIsDescending))
{
throw new DryadLinqException(DryadLinqErrorCode.PartitionKeysAreNotConsistentlyOrdered,
SR.PartitionKeysAreNotConsistentlyOrdered);
}
this.m_isDescending = detectedIsDescending ?? false;
}
else
{
this.m_isDescending = isDescending.GetValueOrDefault();
if (partitionKeys != null &&
!DryadLinqUtil.IsOrdered(partitionKeys, this.m_comparer, this.m_isDescending))
{
throw new DryadLinqException(DryadLinqErrorCode.IsDescendingIsInconsistent,
SR.IsDescendingIsInconsistent);
}
}
}
/// <summary>
/// Private helper to write the current block out to the native stream.
/// - it writes out the current data buffer up to the point it was filled
/// - it releases the current data block back to the native stream code (which owns the lifecycle of read buffers),
/// - then allocated a new buffer from the native stream
/// - and updates the internal read buffer pointer and position members
/// </summary>
private void FlushDataBlock()
{
DataBlockInfo newDataBlockInfo;
if (this.m_curRecordStart <= 16)
{
// The current block is too small for a single record, augment it
if (this.m_curBlockSize == this.m_nextBlockSize)
{
throw new DryadLinqException(DryadLinqErrorCode.RecordSizeMax2GB, SR.RecordSizeMax2GB);
}
newDataBlockInfo = this.m_nativeStream.AllocateDataBlock(this.m_nextBlockSize);
this.m_nextBlockSize = this.m_nextBlockSize * 2;
if (this.m_nextBlockSize < 0)
{
this.m_nextBlockSize = 0x7FFFFFF8;
}
DryadLinqUtil.memcpy(this.m_curDataBlock, newDataBlockInfo.DataBlock, this.m_curRecordEnd);
}
else
{
// Write all the complete records in the block, put the partial record in the new block
newDataBlockInfo = this.m_nativeStream.AllocateDataBlock(this.m_curBlockSize);
DryadLinqUtil.memcpy(this.m_curDataBlock + this.m_curRecordStart,
newDataBlockInfo.DataBlock,
this.m_curRecordEnd - this.m_curRecordStart);
this.m_nativeStream.WriteDataBlock(this.m_curDataBlockInfo.ItemHandle, this.m_curRecordStart);
this.m_numBytesWritten += this.m_curRecordStart;
this.m_curRecordEnd -= this.m_curRecordStart;
this.m_curRecordStart = 0;
}
this.m_nativeStream.ReleaseDataBlock(this.m_curDataBlockInfo.ItemHandle);
this.m_curDataBlockInfo.ItemHandle = IntPtr.Zero;
this.m_curDataBlockInfo = newDataBlockInfo;
this.m_curDataBlock = newDataBlockInfo.DataBlock;
this.m_curBlockSize = newDataBlockInfo.BlockSize;
}
/// <summary>
/// Reads <paramref name="byteCount"/> bytes into <paramref name="destBuffer"/> starting at <paramref name="offset"/>.
/// </summary>
/// <param name="destBuffer">The pre-allocated byte array to read data into.</param>
/// <param name="offset">The starting offset at which to begin reading bytes into <paramref name="destBuffer"/>.</param>
/// <param name="byteCount">The maximum number of bytes to read. </param>
/// <returns>The number of bytes that was actually read.</returns>
public int ReadBytes(byte[] destBuffer, int offset, int byteCount)
{
if (destBuffer == null)
{
throw new ArgumentNullException("destBuffer");
}
if (offset < 0)
{
throw new ArgumentOutOfRangeException("offset");
}
if (byteCount < 0)
{
throw new ArgumentOutOfRangeException("byteCount");
}
if (destBuffer.Length < (offset + byteCount))
{
throw new ArgumentOutOfRangeException("destBuffer",
String.Format(SR.ArrayLengthVsCountAndOffset,
"destBuffer", offset + byteCount,
"offset", "byteCount"));
}
int numBytes = byteCount;
int numBytesRead = 0;
fixed(byte *pBytes = &destBuffer[offset])
{
while (numBytes > 0)
{
int numAvailBytes = this.m_curBlockSize - this.m_curBlockPos;
// Check if there are enough bytes in the read buffer to satisfy the
// caller's request. If so, do the copy, update m_curBlockPos and return
if (numAvailBytes >= numBytes)
{
DryadLinqUtil.memcpy(this.m_curDataBlock + this.m_curBlockPos,
pBytes + numBytesRead,
numBytes);
this.m_curBlockPos += numBytes;
numBytesRead = byteCount;
break;
}
// The remaining data in the read buffer isn't enough to fill the user's request...
// Copy the all the remaining bytes to the destination buffer, and request a
// new read buffer from the stream.
// Note that we don't need to update m_curBlockPos here because the
// GetNextDataBlock call will reset it.
DryadLinqUtil.memcpy(this.m_curDataBlock + this.m_curBlockPos,
pBytes + numBytesRead,
numAvailBytes);
// update numBytes/numBytesRead
numBytes -= numAvailBytes;
numBytesRead += numAvailBytes;
this.GetNextDataBlock();
if (this.m_curBlockSize <= 0)
{
// if the file stream returned an empty buffer it means we are at the
// end of the file. Just return the total number of bytes read, and the
// caller will decide how to handle it.
break;
}
// continue with the loop to keep filling the remaining parts of the
// destination buffer
}
}
return(numBytesRead);
}
