public ChromGroups( IList <IList <int> > chromatogramRequestOrder, IList <ChromKey> chromKeys, float maxRetentionTime, int cycleCount, string cachePath) { RequestOrder = chromatogramRequestOrder; _chromKeys = chromKeys; _maxRetentionTime = maxRetentionTime; _cycleCount = cycleCount; _cachePath = cachePath; if (RequestOrder == null) { return; } // Sanity check. foreach (var group in RequestOrder) { foreach (var chromIndex in group) { Assume.IsTrue(chromIndex >= 0 && chromIndex < _chromKeys.Count); } } // Create array to map a provider id back to the peptide group that contains it. _idToGroupId = new int[_chromKeys.Count]; for (int groupId = 0; groupId < chromatogramRequestOrder.Count; groupId++) { foreach (var id in chromatogramRequestOrder[groupId]) { _idToGroupId[id] = groupId; } } // Decide how groups will be allocated to spill files. long maxSpillFileSize = 0; long estimateSpillFileSize = 0; SpillFile spillFile = new SpillFile(); _spillFiles = new SpillFile[chromatogramRequestOrder.Count]; for (int groupId = 0; groupId < _spillFiles.Length; groupId++) { long maxGroupSize = GetMaxSize(groupId); long estimateGroupSize = EstimateGroupSize(groupId); maxSpillFileSize += maxGroupSize; estimateSpillFileSize += estimateGroupSize; if (maxSpillFileSize > MAX_SPILL_FILE_SIZE || estimateSpillFileSize > TARGET_SPILL_FILE_SIZE) { spillFile = new SpillFile(); maxSpillFileSize = maxGroupSize; estimateSpillFileSize = estimateGroupSize; } _spillFiles[groupId] = spillFile; spillFile.MaxTime = Math.Max(spillFile.MaxTime, GetMaxTime(groupId)); } }
/// <summary> /// Release a chromatogram if its collector is complete (indicated by retention time). /// </summary> /// <returns>-1 if chromatogram is not finished yet.</returns> public int ReleaseChromatogram( int chromatogramIndex, float retentionTime, ChromCollector collector, out TimeIntensities timeIntensities) { int groupIndex = GetGroupIndex(chromatogramIndex); var spillFile = _spillFiles[groupIndex]; // Not done reading yet. if (retentionTime < spillFile.MaxTime || (collector != null && !collector.IsSetTimes)) { timeIntensities = null; return(-1); } // No chromatogram information collected. if (collector == null) { timeIntensities = TimeIntensities.EMPTY; return(0); } if (ReferenceEquals(_cachedSpillFile, spillFile)) { if (spillFile.Stream != null) { if (_bytesFromSpillFile == null || spillFile.Stream.Length != _bytesFromSpillFile.Length) { // Need to reread spill file if more bytes were written since the time it was cached. _cachedSpillFile = null; } } } if (!ReferenceEquals(_cachedSpillFile, spillFile)) { _cachedSpillFile = spillFile; _bytesFromSpillFile = null; var fileStream = spillFile.Stream; if (fileStream != null) { fileStream.Seek(0, SeekOrigin.Begin); _bytesFromSpillFile = new byte[fileStream.Length]; int bytesRead = fileStream.Read(_bytesFromSpillFile, 0, _bytesFromSpillFile.Length); Assume.IsTrue(bytesRead == _bytesFromSpillFile.Length); // TODO: Would be nice to have something that releases spill files progressively, as they are // no longer needed. // spillFile.CloseStream(); } } collector.ReleaseChromatogram(_bytesFromSpillFile, out timeIntensities); return(collector.StatusId); }
/// <summary> /// Release a chromatogram if its collector is complete (indicated by retention time). /// </summary> /// <returns>-1 if chromatogram is not finished yet.</returns> public int ReleaseChromatogram( int chromatogramIndex, float retentionTime, ChromCollector collector, out float[] times, out float[] intensities, out float[] massErrors, out int[] scanIds) { int groupIndex = GetGroupIndex(chromatogramIndex); var spillFile = _spillFiles[groupIndex]; // Not done reading yet. if (retentionTime < spillFile.MaxTime) { times = null; intensities = null; massErrors = null; scanIds = null; return(-1); } // No chromatogram information collected. if (collector == null) { times = EMPTY_FLOAT_ARRAY; intensities = EMPTY_FLOAT_ARRAY; massErrors = null; scanIds = null; return(0); } if (!ReferenceEquals(_cachedSpillFile, spillFile)) { _cachedSpillFile = spillFile; _bytesFromSpillFile = null; var fileStream = spillFile.Stream; if (fileStream != null) { fileStream.Seek(0, SeekOrigin.Begin); _bytesFromSpillFile = new byte[fileStream.Length]; int bytesRead = fileStream.Read(_bytesFromSpillFile, 0, _bytesFromSpillFile.Length); Assume.IsTrue(bytesRead == _bytesFromSpillFile.Length); // TODO: Would be nice to have something that releases spill files progressively, as they are // no longer needed. // spillFile.CloseStream(); } } collector.ReleaseChromatogram(_bytesFromSpillFile, out times, out intensities, out massErrors, out scanIds); return(collector.StatusId); }
SpillFile AcquireFile(Int64 sizeNeeded) { SpillFile f; if (sizeNeeded < 0) { f = new SpillFile(MintFileName()); } else { lock (availableFiles) { var fitFiles = availableFiles.Where(x => x.BufferedSpace >= sizeNeeded); // XXX f = fitFiles.First(); } } return(f); }
public RecvFiberSpillBank(Vertex <T> vertex, int bufferSize) { this.vertex = vertex; this.spillFile = new SpillFile <Pair <S, T> >(System.IO.Path.GetRandomFileName(), bufferSize, new AutoSerializedMessageEncoder <S, T>(1, 1, DummyBufferPool <byte> .Pool, vertex.Stage.InternalGraphManager.Controller.Configuration.SendPageSize, vertex.CodeGenerator), new AutoSerializedMessageDecoder <S, T>(vertex.CodeGenerator), vertex.Stage.InternalGraphManager.Controller.Configuration.SendPageSize, vertex.CodeGenerator.GetSerializer <MessageHeader>()); }
public RecvFiberSpillBank(Vertex <T> shard, int bufferSize) { this.Shard = shard; this.spillFile = new SpillFile <Pair <S, T> >(System.IO.Path.GetRandomFileName(), bufferSize, new AutoSerializedMessageEncoder <S, T>(1, 1, DummyBufferPool <byte> .Pool, shard.Stage.InternalGraphManager.Controller.Configuration.SendPageSize, AutoSerializationMode.OneTimePerMessage), new AutoSerializedMessageDecoder <S, T>(), shard.Stage.InternalGraphManager.Controller.Configuration.SendPageSize); }
SpillFile AcquireFile(Int64 sizeNeeded) { SpillFile f; if (sizeNeeded < 0) { f = new SpillFile(MintFileName()); } else { lock (availableFiles) { var fitFiles = availableFiles.Where(x => x.BufferedSpace >= sizeNeeded); // XXX f = fitFiles.First(); } } return f; }