public unsafe int Write(byte *buffer, int offset, int count) { if (this.m_mode == FileAccess.Read) { throw new DryadLinqException(HpcLinqErrorCode.AttemptToReadFromAWriteStream, SR.AttemptToReadFromAWriteStream); } if (this.m_fstream == null) { this.OpenForWrite(false); } SafeFileHandle handle = this.m_fstream.SafeFileHandle; int size; Int32 * pBlockSize = &size; bool success = HpcLinqNative.WriteFile(handle, buffer, (UInt32)count, (IntPtr)pBlockSize, null); if (!success) { throw new DryadLinqException(HpcLinqErrorCode.WriteFileError, String.Format(SR.WriteFileError, Marshal.GetLastWin32Error())); } this.size += (ulong)size; return(size); }
internal unsafe Int32 GetWriteBuffSize() { MEMORYSTATUSEX memStatus = new MEMORYSTATUSEX(); memStatus.dwLength = (UInt32)sizeof(MEMORYSTATUSEX); UInt64 maxSize = 512 * 1024 * 1024UL; if (HpcLinqNative.GlobalMemoryStatusEx(ref memStatus)) { maxSize = memStatus.ullAvailPhys / 4; } if (this.m_vertexParams.RemoteArch == "i386") { maxSize = Math.Min(maxSize, 1024 * 1024 * 1024UL); } if (this.NumberOfOutputs > 0) { maxSize = maxSize / this.NumberOfOutputs; } UInt64 buffSize = (this.UseLargeBuffer) ? (256 * 1024 * 1024UL) : (1024 * 1024UL); if (buffSize > maxSize) { buffSize = maxSize; } if (buffSize < (8 * 1024UL)) { buffSize = 8 * 1024; } return((Int32)buffSize); }
private unsafe static void DoEviction(object stateInfo) { while (true) { try { MEMORYSTATUSEX memStatus = new MEMORYSTATUSEX(); memStatus.dwLength = (UInt32)sizeof(MEMORYSTATUSEX); HpcLinqNative.GlobalMemoryStatusEx(ref memStatus); if (HpcLinqNative.GlobalMemoryStatusEx(ref memStatus) && memStatus.ullAvailPhys < 4 * 1024 * 1024 * 1024UL) { // Perform eviction only when feeling memory pressure lock (s_cache) { var candidates = s_cache.Where(x => x.Value.RefCount == 0); foreach (var rec in candidates) { s_cache.Remove(rec.Key); } } } } catch (Exception e) { DryadLinqLog.Add("Exception occurred when performing cache eviction: {0}.", e.Message); } } }
internal override unsafe void ReleaseDataBlock(IntPtr itemHandle) { if (itemHandle != IntPtr.Zero) { HpcLinqNative.ReleaseDataBlock(this.m_vertexInfo, itemHandle); } // DryadLinqLog.Add("Released data block {0}.", itemHandle); }
internal void SetInitialWriteSizeHint() { Int64 inputSize = this.GetInputSize(); UInt64 hsize = (inputSize == -1) ? (5 * 1024 * 1024 * 1024UL) : (UInt64)inputSize; hsize /= this.NumberOfOutputs; for (UInt32 i = 0; i < this.NumberOfOutputs; i++) { HpcLinqNative.SetInitialSizeHint(this.m_nativeHandle, i, hsize); } }
internal override unsafe Int64 GetTotalLength() { if (this.m_isInput) { return(HpcLinqNative.GetExpectedLength(this.m_vertexInfo, this.m_portNum)); } else { throw new NotImplementedException(); } }
internal override unsafe bool WriteDataBlock(IntPtr itemHandle, Int32 numBytesToWrite) { byte *dataBlock = (byte *)itemHandle; if (this.m_compressionScheme == DscCompressionScheme.None) { Int32 numBytesWritten = 0; Int32 remainingBytes = numBytesToWrite; while (remainingBytes > 0) { Int32 *pNumBytesWritten = &numBytesWritten; bool success = HpcLinqNative.WriteFile(this.m_fhandle, dataBlock, (UInt32)remainingBytes, (IntPtr)pNumBytesWritten, null); if (!success) { throw new DryadLinqException(HpcLinqErrorCode.WriteFileError, String.Format(SR.WriteFileError, Marshal.GetLastWin32Error())); } dataBlock += numBytesWritten; remainingBytes -= numBytesWritten; } } else { if (this.m_compressStream == null) { if (this.m_compressionScheme == DscCompressionScheme.Gzip) { this.m_compressStream = new GZipStream(this.m_fstream, CompressionMode.Compress); } else { throw new DryadLinqException(HpcLinqErrorCode.UnknownCompressionScheme, SR.UnknownCompressionScheme); } } // YY: Made an extra copy here. Could do better. byte[] buffer = new byte[numBytesToWrite]; fixed(byte *pBuffer = buffer) { HpcLinqUtil.memcpy(dataBlock, pBuffer, numBytesToWrite); } this.m_compressStream.Write(buffer, 0, numBytesToWrite); } return(true); }
internal override void Close() { if (!this.m_isClosed) { this.m_isClosed = true; this.Flush(); HpcLinqNative.Close(this.m_vertexInfo, this.m_portNum); string ctype = (this.m_isInput) ? "Input" : "Output"; DryadLinqLog.Add(ctype + " channel {0} was closed.", this.m_portNum); } GC.SuppressFinalize(this); }
internal override unsafe Int64 GetTotalLength() { Int64 totalLen; bool success = HpcLinqNative.GetFileSizeEx(this.m_fhandle, out totalLen); if (!success) { throw new DryadLinqException(HpcLinqErrorCode.GetFileSizeError, String.Format(SR.GetFileSizeError, Marshal.GetLastWin32Error())); } return(totalLen); }
internal override unsafe string GetURI() { IntPtr uriPtr; if (this.m_isInput) { uriPtr = HpcLinqNative.GetInputChannelURI(this.m_vertexInfo, this.m_portNum); } else { uriPtr = HpcLinqNative.GetOutputChannelURI(this.m_vertexInfo, this.m_portNum); } return(Marshal.PtrToStringAnsi(uriPtr)); }
internal override unsafe DataBlockInfo AllocateDataBlock(Int32 size) { DataBlockInfo blockInfo; blockInfo.itemHandle = HpcLinqNative.AllocateDataBlock(this.m_vertexInfo, size, &blockInfo.dataBlock); blockInfo.blockSize = size; if (blockInfo.itemHandle == IntPtr.Zero) { throw new DryadLinqException(HpcLinqErrorCode.FailedToAllocateNewNativeBuffer, String.Format(SR.FailedToAllocateNewNativeBuffer, size)); } // DryadLinqLog.Add("Allocated data block {0} of {1} bytes.", blockInfo.itemHandle, size); return(blockInfo); }
internal Int64 GetInputSize() { Int64 totalSize = 0; for (UInt32 i = 0; i < this.m_numberOfInputs; i++) { Int64 channelSize = HpcLinqNative.GetExpectedLength(this.NativeHandle, i); if (channelSize == -1) { return(-1); } totalSize += channelSize; } return(totalSize); }
internal override unsafe DataBlockInfo ReadDataBlock() { DataBlockInfo blockInfo; blockInfo.dataBlock = (byte *)Marshal.AllocHGlobal(DefaultBuffSize); blockInfo.itemHandle = (IntPtr)blockInfo.dataBlock; if (this.m_compressionScheme == DscCompressionScheme.None) { Int32 *pBlockSize = &blockInfo.blockSize; bool success = HpcLinqNative.ReadFile(this.m_fhandle, blockInfo.dataBlock, DefaultBuffSize, (IntPtr)pBlockSize, null); if (!success) { throw new DryadLinqException(HpcLinqErrorCode.ReadFileError, String.Format(SR.ReadFileError, Marshal.GetLastWin32Error())); } } else { if (this.m_compressStream == null) { if (this.m_compressionScheme == DscCompressionScheme.Gzip) { this.m_compressStream = new GZipStream(this.m_fstream, CompressionMode.Decompress); } else { throw new DryadLinqException(HpcLinqErrorCode.UnknownCompressionScheme, SR.UnknownCompressionScheme); } } // YY: Made an extra copy here. Could do better. byte[] buffer = new byte[DefaultBuffSize]; blockInfo.blockSize = this.m_compressStream.Read(buffer, 0, DefaultBuffSize); fixed(byte *pBuffer = buffer) { HpcLinqUtil.memcpy(pBuffer, blockInfo.dataBlock, blockInfo.blockSize); } } return(blockInfo); }
internal override unsafe DataBlockInfo ReadDataBlock() { DataBlockInfo blockInfo; Int32 errorCode = 0; blockInfo.itemHandle = HpcLinqNative.ReadDataBlock(this.m_vertexInfo, this.m_portNum, &blockInfo.dataBlock, &blockInfo.blockSize, &errorCode); if (errorCode != 0) { HpcLinqVertexEnv.ErrorCode = errorCode; throw new DryadLinqException(HpcLinqErrorCode.FailedToReadFromInputChannel, String.Format(SR.FailedToReadFromInputChannel, this.m_portNum, errorCode)); } return(blockInfo); }
internal override unsafe bool WriteDataBlock(IntPtr itemHandle, Int32 numBytesToWrite) { bool success = true; if (numBytesToWrite > 0) { success = HpcLinqNative.WriteDataBlock(this.m_vertexInfo, this.m_portNum, itemHandle, numBytesToWrite); if (!success) { throw new DryadLinqException(HpcLinqErrorCode.FailedToWriteToOutputChannel, String.Format(SR.FailedToWriteToOutputChannel, this.m_portNum)); } } return(success); }
public HpcLinqVertexEnv(string args, HpcLinqVertexParams vertexParams) { this.m_argList = args.Split('|'); this.m_nativeHandle = new IntPtr(Int64.Parse(this.m_argList[0], NumberStyles.HexNumber)); this.m_numberOfInputs = HpcLinqNative.GetNumOfInputs(this.m_nativeHandle); this.m_numberOfOutputs = HpcLinqNative.GetNumOfOutputs(this.m_nativeHandle); this.m_nextInputPort = 0; this.m_nextOutputPort = 0; this.m_vertexParams = vertexParams; this.m_useLargeBuffer = vertexParams.UseLargeBuffer; this.m_keepInputPortOrder = vertexParams.KeepInputPortOrder; this.m_multiThreading = vertexParams.MultiThreading; if (this.m_numberOfOutputs > 0) { this.SetInitialWriteSizeHint(); } Debug.Assert(vertexParams.InputArity <= this.m_numberOfInputs); Debug.Assert(vertexParams.OutputArity <= this.m_numberOfOutputs); }
internal unsafe int Read(byte *buffer, int bufferSize) { int totalBytesRead = 0; do { SafeFileHandle handle; if (this.m_fstream == null) { this.OpenForRead(); } if (this.m_atEOF) { break; } handle = this.m_fstream.SafeFileHandle; int size = 0; Int32 *pBlockSize = &size; bool success = HpcLinqNative.ReadFile(handle, buffer, (UInt32)bufferSize, (IntPtr)pBlockSize, null); if (!success) { throw new DryadLinqException(HpcLinqErrorCode.ReadFileError, String.Format(SR.ReadFileError, Marshal.GetLastWin32Error())); } totalBytesRead += size; if (size == 0) { this.m_fstream.Close(); this.m_fstream = null; } } while (totalBytesRead == 0 && !this.m_atEOF); return(totalBytesRead); }
public static IEnumerable <K> Phase1Sampling <T, K>(IEnumerable <T> source, Func <T, K> keySelector, HpcLinqVertexEnv denv) { // note: vertexID is constant for each repetition of a specific vertex (eg in fail-and-retry scenarios) // this is very good as it ensure the sampling is idempotent w.r.t. retries. long vertexID = HpcLinqNative.GetVertexId(denv.NativeHandle); int seed = unchecked ((int)(vertexID)); long nEmitted = 0; Random rdm = new Random(seed); List <K> allSoFar = new List <K>(); List <K> samples = new List <K>(); // try to collect 10 samples, but keep all the records just in case IEnumerator <T> sourceEnumerator = source.GetEnumerator(); while (sourceEnumerator.MoveNext()) { T elem = sourceEnumerator.Current; K key = keySelector(elem); allSoFar.Add(key); if (rdm.NextDouble() < SAMPLE_RATE) { samples.Add(key); if (samples.Count >= 10) { break; } } } if (samples.Count >= 10) { // we have lots of samples.. emit them and continue sampling allSoFar = null; // not needed. foreach (K key in samples) { yield return(key); nEmitted++; } while (sourceEnumerator.MoveNext()) { T elem = sourceEnumerator.Current; if (rdm.NextDouble() < SAMPLE_RATE) { yield return(keySelector(elem)); nEmitted++; } } } else { // sampling didn't produce much, so emit all the records instead. DryadLinqLog.Add("Sampling produced only {0} records. Emitting all records instead.", samples.Count()); Debug.Assert(sourceEnumerator.MoveNext() == false, "The source enumerator wasn't finished"); samples = null; // the samples list is not needed. foreach (K key in allSoFar) { yield return(key); nEmitted++; } } DryadLinqLog.Add("Stage1 sampling: num keys emitted = {0}", nEmitted); }
internal override void Flush() { HpcLinqNative.Flush(this.m_vertexInfo, this.m_portNum); }