public MutateHelper(DiskRangeList head) : base(-1, -1) { Debug.Assert(head != null); Debug.Assert(head.prev == null); this.next = head; head.prev = this; }
public DiskRangeList extract() { DiskRangeList result = head; head = null; return(result); }
public DiskRangeList extract() { DiskRangeList result = this.next; Debug.Assert(result != null); this.next = result.prev = null; return(result); }
/** * Inserts an element after current in the list. * @returns the new element. * */ public DiskRangeList insertAfter(DiskRangeList other) { other.next = this.next; other.prev = this; if (this.next != null) { this.next.prev = other; } this.next = other; return(other); }
/** Removes an element after current from the list. */ public void removeAfter() { DiskRangeList other = this.next; this.next = other.next; if (this.next != null) { this.next.prev = this; } other.next = other.prev = null; }
/** * Inserts an element after current in the list. * @returns the new element. * */ public DiskRangeList insertAfter(DiskRangeList other) { other.next = this.next; other.prev = this; if (this.next != null) { this.next.prev = other; } this.next = other; return other; }
/** Removes the current element from the list. */ public void removeSelf() { if (this.prev != null) { this.prev.next = this.next; } if (this.next != null) { this.next.prev = this.prev; } this.next = this.prev = null; }
// @VisibleForTesting public int listSize() { int result = 1; DiskRangeList current = this.next; while (current != null) { ++result; current = current.next; } return(result); }
public long getTotalLength() { long totalLength = getLength(); DiskRangeList current = next; while (current != null) { totalLength += current.getLength(); current = current.next; } return(totalLength); }
/** * Inserts an intersecting range before current in the list and adjusts offset accordingly. * @returns the new element. */ public DiskRangeList insertPartBefore(DiskRangeList other) { Debug.Assert(other.end >= this.offset); this.offset = other.end; other.prev = this.prev; other.next = this; if (this.prev != null) { this.prev.next = other; } this.prev = other; return(other); }
// @VisibleForTesting public DiskRangeList[] listToArray() { DiskRangeList[] result = new DiskRangeList[listSize()]; int i = 0; DiskRangeList current = this.next; while (current != null) { result[i] = current; ++i; current = current.next; } return(result); }
/** Replaces this element with another in the list; returns the new element. */ public DiskRangeList replaceSelfWith(DiskRangeList other) { other.prev = this.prev; other.next = this.next; if (this.prev != null) { this.prev.next = other; } if (this.next != null) { this.next.prev = other; } this.next = this.prev = null; return(other); }
public static void addRgFilteredStreamToRanges(OrcProto.Stream stream, bool[] includedRowGroups, bool isCompressed, OrcProto.RowIndex index, OrcProto.ColumnEncoding encoding, OrcProto.Type type, int compressionSize, bool hasNull, long offset, long length, DiskRangeList.CreateHelper list, bool doMergeBuffers) { for (int group = 0; group < includedRowGroups.Length; ++group) { if (!includedRowGroups[group]) continue; int posn = getIndexPosition( encoding.Kind, type.Kind, stream.Kind, isCompressed, hasNull); long start = (long)index.EntryList[group].PositionsList[posn]; long nextGroupOffset; bool isLast = group == (includedRowGroups.Length - 1); nextGroupOffset = isLast ? length : (int)index.EntryList[group + 1].PositionsList[posn]; start += offset; long end = offset + estimateRgEndOffset( isCompressed, isLast, nextGroupOffset, length, compressionSize); list.addOrMerge(start, end, doMergeBuffers, true); } }
public void addOrMerge(long offset, long end, bool doMerge, bool doLogNew) { if (doMerge && tail != null && tail.merge(offset, end)) { return; } if (doLogNew) { LOG.info("Creating new range; last range (which can include some previous adds) was " + tail); } DiskRangeList node = new DiskRangeList(offset, end); if (tail == null) { head = tail = node; } else { tail = tail.insertAfter(node); } }
internal static List<DiskRange> getStreamBuffers(DiskRangeList range, long offset, long length) { // This assumes sorted ranges (as do many other parts of ORC code. List<DiskRange> buffers = new List<DiskRange>(); if (length == 0) return buffers; long streamEnd = offset + length; bool inRange = false; while (range != null) { if (!inRange) { if (range.getEnd() <= offset) { range = range.next; continue; // Skip until we are in range. } inRange = true; if (range.getOffset() < offset) { // Partial first buffer, add a slice of it. buffers.Add(range.sliceAndShift(offset, Math.Min(streamEnd, range.getEnd()), -offset)); if (range.getEnd() >= streamEnd) break; // Partial first buffer is also partial last buffer. range = range.next; continue; } } else if (range.getOffset() >= streamEnd) { break; } if (range.getEnd() > streamEnd) { // Partial last buffer (may also be the first buffer), add a slice of it. buffers.Add(range.sliceAndShift(range.getOffset(), streamEnd, -offset)); break; } // Buffer that belongs entirely to one stream. // TODO: ideally we would want to reuse the object and remove it from the list, but we cannot // because bufferChunks is also used by clearStreams for zcr. Create a useless dup. buffers.Add(range.sliceAndShift(range.getOffset(), range.getEnd(), -offset)); if (range.getEnd() == streamEnd) break; range = range.next; } return buffers; }
/** * Read the list of ranges from the file. * @param file the file to read * @param base the base of the stripe * @param ranges the disk ranges within the stripe to read * @return the bytes read for each disk range, which is the same length as * ranges * @ */ static DiskRangeList readDiskRanges(Stream file, long @base, DiskRangeList range, bool doForceDirect) { if (range == null) return null; DiskRangeList prev = range.prev; if (prev == null) { prev = new DiskRangeList.MutateHelper(range); } while (range != null) { if (range.hasData()) { range = range.next; continue; } int len = (int)(range.getEnd() - range.getOffset()); long off = range.getOffset(); if (doForceDirect) { file.Seek(@base + off, SeekOrigin.Current); ByteBuffer directBuf = ByteBuffer.allocateDirect(len); readDirect(file, len, directBuf); range = range.replaceSelfWith(new RecordReaderImpl.BufferChunk(directBuf, range.getOffset())); } else { byte[] buffer = new byte[len]; file.readFully((@base + off), buffer, 0, buffer.Length); range = range.replaceSelfWith(new RecordReaderImpl.BufferChunk(ByteBuffer.wrap(buffer), range.getOffset())); } range = range.next; } return prev.next; }
public DiskRangeList readFileData( DiskRangeList range, long baseOffset, bool doForceDirect) { return RecordReaderUtils.readDiskRanges(file, baseOffset, range, doForceDirect); }
public static void addEntireStreamToRanges( long offset, long length, DiskRangeList.CreateHelper list, bool doMergeBuffers) { list.addOrMerge(offset, offset + length, doMergeBuffers, false); }
/** Replaces this element with another in the list; returns the new element. */ public DiskRangeList replaceSelfWith(DiskRangeList other) { other.prev = this.prev; other.next = this.next; if (this.prev != null) { this.prev.next = other; } if (this.next != null) { this.next.prev = other; } this.next = this.prev = null; return other; }
public void addOrMerge(long offset, long end, bool doMerge, bool doLogNew) { if (doMerge && tail != null && tail.merge(offset, end)) return; if (doLogNew) { LOG.info("Creating new range; last range (which can include some previous adds) was " + tail); } DiskRangeList node = new DiskRangeList(offset, end); if (tail == null) { head = tail = node; } else { tail = tail.insertAfter(node); } }
// @VisibleForTesting public DiskRangeList[] listToArray() { DiskRangeList[] result = new DiskRangeList[listSize()]; int i = 0; DiskRangeList current = this.next; while (current != null) { result[i] = current; ++i; current = current.next; } return result; }
public DiskRangeList extract() { DiskRangeList result = head; head = null; return result; }
/** * Inserts an intersecting range after current in the list and adjusts offset accordingly. * @returns the new element. */ public DiskRangeList insertPartAfter(DiskRangeList other) { Debug.Assert(other.offset <= this.end); this.end = other.offset; return(insertAfter(other)); }
/** * Inserts an intersecting range after current in the list and adjusts offset accordingly. * @returns the new element. */ public DiskRangeList insertPartAfter(DiskRangeList other) { Debug.Assert(other.offset <= this.end); this.end = other.offset; return insertAfter(other); }
private static DiskRangeList diskRanges(params int[] points) { DiskRangeList head = null, tail = null; for (int i = 0; i < points.Length; i += 2) { DiskRangeList range = new DiskRangeList(points[i], points[i + 1]); if (tail == null) { head = tail = range; } else { tail = tail.insertAfter(range); } } return head; }
/** * Inserts an intersecting range before current in the list and adjusts offset accordingly. * @returns the new element. */ public DiskRangeList insertPartBefore(DiskRangeList other) { Debug.Assert(other.end >= this.offset); this.offset = other.end; other.prev = this.prev; other.next = this; if (this.prev != null) { this.prev.next = other; } this.prev = other; return other; }
/** * Build a string representation of a list of disk ranges. * @param ranges ranges to stringify * @return the resulting string */ public static string stringifyDiskRanges(DiskRangeList range) { StringBuilder buffer = new StringBuilder(); buffer.Append("["); bool isFirst = true; while (range != null) { if (!isFirst) { buffer.Append(", {"); } else { buffer.Append("{"); } isFirst = false; buffer.Append(range.ToString()); buffer.Append("}"); range = range.next; } buffer.Append("]"); return buffer.ToString(); }