public unsafe int Write(byte *buffer, int offset, int count)
{
if (this.m_mode == FileAccess.Read)
{
throw new DryadLinqException(HpcLinqErrorCode.AttemptToReadFromAWriteStream,
SR.AttemptToReadFromAWriteStream);
}
if (this.m_fstream == null)
{
this.OpenForWrite(false);
}
SafeFileHandle handle = this.m_fstream.SafeFileHandle;
int size;
Int32 * pBlockSize = &size;
bool success = HpcLinqNative.WriteFile(handle, buffer, (UInt32)count, (IntPtr)pBlockSize, null);
if (!success)
{
throw new DryadLinqException(HpcLinqErrorCode.WriteFileError,
String.Format(SR.WriteFileError, Marshal.GetLastWin32Error()));
}
this.size += (ulong)size;
return(size);
}
internal unsafe Int32 GetWriteBuffSize()
{
MEMORYSTATUSEX memStatus = new MEMORYSTATUSEX();
memStatus.dwLength = (UInt32)sizeof(MEMORYSTATUSEX);
UInt64 maxSize = 512 * 1024 * 1024UL;
if (HpcLinqNative.GlobalMemoryStatusEx(ref memStatus))
{
maxSize = memStatus.ullAvailPhys / 4;
}
if (this.m_vertexParams.RemoteArch == "i386")
{
maxSize = Math.Min(maxSize, 1024 * 1024 * 1024UL);
}
if (this.NumberOfOutputs > 0)
{
maxSize = maxSize / this.NumberOfOutputs;
}
UInt64 buffSize = (this.UseLargeBuffer) ? (256 * 1024 * 1024UL) : (1024 * 1024UL);
if (buffSize > maxSize)
{
buffSize = maxSize;
}
if (buffSize < (8 * 1024UL))
{
buffSize = 8 * 1024;
}
return((Int32)buffSize);
}
private unsafe static void DoEviction(object stateInfo)
{
while (true)
{
try
{
MEMORYSTATUSEX memStatus = new MEMORYSTATUSEX();
memStatus.dwLength = (UInt32)sizeof(MEMORYSTATUSEX);
HpcLinqNative.GlobalMemoryStatusEx(ref memStatus);
if (HpcLinqNative.GlobalMemoryStatusEx(ref memStatus) &&
memStatus.ullAvailPhys < 4 * 1024 * 1024 * 1024UL)
{
// Perform eviction only when feeling memory pressure
lock (s_cache)
{
var candidates = s_cache.Where(x => x.Value.RefCount == 0);
foreach (var rec in candidates)
{
s_cache.Remove(rec.Key);
}
}
}
}
catch (Exception e)
{
DryadLinqLog.Add("Exception occurred when performing cache eviction: {0}.",
e.Message);
}
}
}
internal override unsafe void ReleaseDataBlock(IntPtr itemHandle)
{
if (itemHandle != IntPtr.Zero)
{
HpcLinqNative.ReleaseDataBlock(this.m_vertexInfo, itemHandle);
}
// DryadLinqLog.Add("Released data block {0}.", itemHandle);
}
internal void SetInitialWriteSizeHint()
{
Int64 inputSize = this.GetInputSize();
UInt64 hsize = (inputSize == -1) ? (5 * 1024 * 1024 * 1024UL) : (UInt64)inputSize;
hsize /= this.NumberOfOutputs;
for (UInt32 i = 0; i < this.NumberOfOutputs; i++)
{
HpcLinqNative.SetInitialSizeHint(this.m_nativeHandle, i, hsize);
}
}
internal override unsafe Int64 GetTotalLength()
{
if (this.m_isInput)
{
return(HpcLinqNative.GetExpectedLength(this.m_vertexInfo, this.m_portNum));
}
else
{
throw new NotImplementedException();
}
}
internal override unsafe bool WriteDataBlock(IntPtr itemHandle, Int32 numBytesToWrite)
{
byte *dataBlock = (byte *)itemHandle;
if (this.m_compressionScheme == DscCompressionScheme.None)
{
Int32 numBytesWritten = 0;
Int32 remainingBytes = numBytesToWrite;
while (remainingBytes > 0)
{
Int32 *pNumBytesWritten = &numBytesWritten;
bool success = HpcLinqNative.WriteFile(this.m_fhandle,
dataBlock,
(UInt32)remainingBytes,
(IntPtr)pNumBytesWritten,
null);
if (!success)
{
throw new DryadLinqException(HpcLinqErrorCode.WriteFileError,
String.Format(SR.WriteFileError,
Marshal.GetLastWin32Error()));
}
dataBlock += numBytesWritten;
remainingBytes -= numBytesWritten;
}
}
else
{
if (this.m_compressStream == null)
{
if (this.m_compressionScheme == DscCompressionScheme.Gzip)
{
this.m_compressStream = new GZipStream(this.m_fstream,
CompressionMode.Compress);
}
else
{
throw new DryadLinqException(HpcLinqErrorCode.UnknownCompressionScheme,
SR.UnknownCompressionScheme);
}
}
// YY: Made an extra copy here. Could do better.
byte[] buffer = new byte[numBytesToWrite];
fixed(byte *pBuffer = buffer)
{
HpcLinqUtil.memcpy(dataBlock, pBuffer, numBytesToWrite);
}
this.m_compressStream.Write(buffer, 0, numBytesToWrite);
}
return(true);
}
internal override void Close()
{
if (!this.m_isClosed)
{
this.m_isClosed = true;
this.Flush();
HpcLinqNative.Close(this.m_vertexInfo, this.m_portNum);
string ctype = (this.m_isInput) ? "Input" : "Output";
DryadLinqLog.Add(ctype + " channel {0} was closed.", this.m_portNum);
}
GC.SuppressFinalize(this);
}
internal override unsafe Int64 GetTotalLength()
{
Int64 totalLen;
bool success = HpcLinqNative.GetFileSizeEx(this.m_fhandle, out totalLen);
if (!success)
{
throw new DryadLinqException(HpcLinqErrorCode.GetFileSizeError,
String.Format(SR.GetFileSizeError,
Marshal.GetLastWin32Error()));
}
return(totalLen);
}
internal override unsafe string GetURI()
{
IntPtr uriPtr;
if (this.m_isInput)
{
uriPtr = HpcLinqNative.GetInputChannelURI(this.m_vertexInfo, this.m_portNum);
}
else
{
uriPtr = HpcLinqNative.GetOutputChannelURI(this.m_vertexInfo, this.m_portNum);
}
return(Marshal.PtrToStringAnsi(uriPtr));
}
internal override unsafe DataBlockInfo AllocateDataBlock(Int32 size)
{
DataBlockInfo blockInfo;
blockInfo.itemHandle =
HpcLinqNative.AllocateDataBlock(this.m_vertexInfo, size, &blockInfo.dataBlock);
blockInfo.blockSize = size;
if (blockInfo.itemHandle == IntPtr.Zero)
{
throw new DryadLinqException(HpcLinqErrorCode.FailedToAllocateNewNativeBuffer,
String.Format(SR.FailedToAllocateNewNativeBuffer, size));
}
// DryadLinqLog.Add("Allocated data block {0} of {1} bytes.", blockInfo.itemHandle, size);
return(blockInfo);
}
internal Int64 GetInputSize()
{
Int64 totalSize = 0;
for (UInt32 i = 0; i < this.m_numberOfInputs; i++)
{
Int64 channelSize = HpcLinqNative.GetExpectedLength(this.NativeHandle, i);
if (channelSize == -1)
{
return(-1);
}
totalSize += channelSize;
}
return(totalSize);
}
internal override unsafe DataBlockInfo ReadDataBlock()
{
DataBlockInfo blockInfo;
blockInfo.dataBlock = (byte *)Marshal.AllocHGlobal(DefaultBuffSize);
blockInfo.itemHandle = (IntPtr)blockInfo.dataBlock;
if (this.m_compressionScheme == DscCompressionScheme.None)
{
Int32 *pBlockSize = &blockInfo.blockSize;
bool success = HpcLinqNative.ReadFile(this.m_fhandle,
blockInfo.dataBlock,
DefaultBuffSize,
(IntPtr)pBlockSize,
null);
if (!success)
{
throw new DryadLinqException(HpcLinqErrorCode.ReadFileError,
String.Format(SR.ReadFileError,
Marshal.GetLastWin32Error()));
}
}
else
{
if (this.m_compressStream == null)
{
if (this.m_compressionScheme == DscCompressionScheme.Gzip)
{
this.m_compressStream = new GZipStream(this.m_fstream,
CompressionMode.Decompress);
}
else
{
throw new DryadLinqException(HpcLinqErrorCode.UnknownCompressionScheme,
SR.UnknownCompressionScheme);
}
}
// YY: Made an extra copy here. Could do better.
byte[] buffer = new byte[DefaultBuffSize];
blockInfo.blockSize = this.m_compressStream.Read(buffer, 0, DefaultBuffSize);
fixed(byte *pBuffer = buffer)
{
HpcLinqUtil.memcpy(pBuffer, blockInfo.dataBlock, blockInfo.blockSize);
}
}
return(blockInfo);
}
internal override unsafe DataBlockInfo ReadDataBlock()
{
DataBlockInfo blockInfo;
Int32 errorCode = 0;
blockInfo.itemHandle = HpcLinqNative.ReadDataBlock(this.m_vertexInfo,
this.m_portNum,
&blockInfo.dataBlock,
&blockInfo.blockSize,
&errorCode);
if (errorCode != 0)
{
HpcLinqVertexEnv.ErrorCode = errorCode;
throw new DryadLinqException(HpcLinqErrorCode.FailedToReadFromInputChannel,
String.Format(SR.FailedToReadFromInputChannel,
this.m_portNum, errorCode));
}
return(blockInfo);
}
internal override unsafe bool WriteDataBlock(IntPtr itemHandle, Int32 numBytesToWrite)
{
bool success = true;
if (numBytesToWrite > 0)
{
success = HpcLinqNative.WriteDataBlock(this.m_vertexInfo,
this.m_portNum,
itemHandle,
numBytesToWrite);
if (!success)
{
throw new DryadLinqException(HpcLinqErrorCode.FailedToWriteToOutputChannel,
String.Format(SR.FailedToWriteToOutputChannel,
this.m_portNum));
}
}
return(success);
}
public HpcLinqVertexEnv(string args, HpcLinqVertexParams vertexParams)
{
this.m_argList = args.Split('|');
this.m_nativeHandle = new IntPtr(Int64.Parse(this.m_argList[0], NumberStyles.HexNumber));
this.m_numberOfInputs = HpcLinqNative.GetNumOfInputs(this.m_nativeHandle);
this.m_numberOfOutputs = HpcLinqNative.GetNumOfOutputs(this.m_nativeHandle);
this.m_nextInputPort = 0;
this.m_nextOutputPort = 0;
this.m_vertexParams = vertexParams;
this.m_useLargeBuffer = vertexParams.UseLargeBuffer;
this.m_keepInputPortOrder = vertexParams.KeepInputPortOrder;
this.m_multiThreading = vertexParams.MultiThreading;
if (this.m_numberOfOutputs > 0)
{
this.SetInitialWriteSizeHint();
}
Debug.Assert(vertexParams.InputArity <= this.m_numberOfInputs);
Debug.Assert(vertexParams.OutputArity <= this.m_numberOfOutputs);
}
internal unsafe int Read(byte *buffer, int bufferSize)
{
int totalBytesRead = 0;
do
{
SafeFileHandle handle;
if (this.m_fstream == null)
{
this.OpenForRead();
}
if (this.m_atEOF)
{
break;
}
handle = this.m_fstream.SafeFileHandle;
int size = 0;
Int32 *pBlockSize = &size;
bool success = HpcLinqNative.ReadFile(handle, buffer, (UInt32)bufferSize, (IntPtr)pBlockSize, null);
if (!success)
{
throw new DryadLinqException(HpcLinqErrorCode.ReadFileError,
String.Format(SR.ReadFileError, Marshal.GetLastWin32Error()));
}
totalBytesRead += size;
if (size == 0)
{
this.m_fstream.Close();
this.m_fstream = null;
}
} while (totalBytesRead == 0 && !this.m_atEOF);
return(totalBytesRead);
}
public static IEnumerable <K> Phase1Sampling <T, K>(IEnumerable <T> source,
Func <T, K> keySelector,
HpcLinqVertexEnv denv)
{
// note: vertexID is constant for each repetition of a specific vertex (eg in fail-and-retry scenarios)
// this is very good as it ensure the sampling is idempotent w.r.t. retries.
long vertexID = HpcLinqNative.GetVertexId(denv.NativeHandle);
int seed = unchecked ((int)(vertexID));
long nEmitted = 0;
Random rdm = new Random(seed);
List <K> allSoFar = new List <K>();
List <K> samples = new List <K>();
// try to collect 10 samples, but keep all the records just in case
IEnumerator <T> sourceEnumerator = source.GetEnumerator();
while (sourceEnumerator.MoveNext())
{
T elem = sourceEnumerator.Current;
K key = keySelector(elem);
allSoFar.Add(key);
if (rdm.NextDouble() < SAMPLE_RATE)
{
samples.Add(key);
if (samples.Count >= 10)
{
break;
}
}
}
if (samples.Count >= 10)
{
// we have lots of samples.. emit them and continue sampling
allSoFar = null; // not needed.
foreach (K key in samples)
{
yield return(key);
nEmitted++;
}
while (sourceEnumerator.MoveNext())
{
T elem = sourceEnumerator.Current;
if (rdm.NextDouble() < SAMPLE_RATE)
{
yield return(keySelector(elem));
nEmitted++;
}
}
}
else
{
// sampling didn't produce much, so emit all the records instead.
DryadLinqLog.Add("Sampling produced only {0} records. Emitting all records instead.", samples.Count());
Debug.Assert(sourceEnumerator.MoveNext() == false, "The source enumerator wasn't finished");
samples = null; // the samples list is not needed.
foreach (K key in allSoFar)
{
yield return(key);
nEmitted++;
}
}
DryadLinqLog.Add("Stage1 sampling: num keys emitted = {0}", nEmitted);
}
internal override void Flush()
{
HpcLinqNative.Flush(this.m_vertexInfo, this.m_portNum);
}
