public MutateHelper(DiskRangeList head) : base(-1, -1)
{
Debug.Assert(head != null);
Debug.Assert(head.prev == null);
this.next = head;
head.prev = this;
}
public DiskRangeList extract()
{
DiskRangeList result = head;
head = null;
return(result);
}
public DiskRangeList extract()
{
DiskRangeList result = this.next;
Debug.Assert(result != null);
this.next = result.prev = null;
return(result);
}
/**
* Inserts an element after current in the list.
* @returns the new element.
* */
public DiskRangeList insertAfter(DiskRangeList other)
{
other.next = this.next;
other.prev = this;
if (this.next != null)
{
this.next.prev = other;
}
this.next = other;
return(other);
}
/** Removes an element after current from the list. */
public void removeAfter()
{
DiskRangeList other = this.next;
this.next = other.next;
if (this.next != null)
{
this.next.prev = this;
}
other.next = other.prev = null;
}
/**
* Inserts an element after current in the list.
* @returns the new element.
* */
public DiskRangeList insertAfter(DiskRangeList other)
{
other.next = this.next;
other.prev = this;
if (this.next != null)
{
this.next.prev = other;
}
this.next = other;
return other;
}
/** Removes the current element from the list. */
public void removeSelf()
{
if (this.prev != null)
{
this.prev.next = this.next;
}
if (this.next != null)
{
this.next.prev = this.prev;
}
this.next = this.prev = null;
}
// @VisibleForTesting
public int listSize()
{
int result = 1;
DiskRangeList current = this.next;
while (current != null)
{
++result;
current = current.next;
}
return(result);
}
public long getTotalLength()
{
long totalLength = getLength();
DiskRangeList current = next;
while (current != null)
{
totalLength += current.getLength();
current = current.next;
}
return(totalLength);
}
/**
* Inserts an intersecting range before current in the list and adjusts offset accordingly.
* @returns the new element.
*/
public DiskRangeList insertPartBefore(DiskRangeList other)
{
Debug.Assert(other.end >= this.offset);
this.offset = other.end;
other.prev = this.prev;
other.next = this;
if (this.prev != null)
{
this.prev.next = other;
}
this.prev = other;
return(other);
}
// @VisibleForTesting
public DiskRangeList[] listToArray()
{
DiskRangeList[] result = new DiskRangeList[listSize()];
int i = 0;
DiskRangeList current = this.next;
while (current != null)
{
result[i] = current;
++i;
current = current.next;
}
return(result);
}
/** Replaces this element with another in the list; returns the new element. */
public DiskRangeList replaceSelfWith(DiskRangeList other)
{
other.prev = this.prev;
other.next = this.next;
if (this.prev != null)
{
this.prev.next = other;
}
if (this.next != null)
{
this.next.prev = other;
}
this.next = this.prev = null;
return(other);
}
public static void addRgFilteredStreamToRanges(OrcProto.Stream stream,
bool[] includedRowGroups, bool isCompressed, OrcProto.RowIndex index,
OrcProto.ColumnEncoding encoding, OrcProto.Type type, int compressionSize, bool hasNull,
long offset, long length, DiskRangeList.CreateHelper list, bool doMergeBuffers)
{
for (int group = 0; group < includedRowGroups.Length; ++group)
{
if (!includedRowGroups[group]) continue;
int posn = getIndexPosition(
encoding.Kind, type.Kind, stream.Kind, isCompressed, hasNull);
long start = (long)index.EntryList[group].PositionsList[posn];
long nextGroupOffset;
bool isLast = group == (includedRowGroups.Length - 1);
nextGroupOffset = isLast ? length : (int)index.EntryList[group + 1].PositionsList[posn];
start += offset;
long end = offset + estimateRgEndOffset(
isCompressed, isLast, nextGroupOffset, length, compressionSize);
list.addOrMerge(start, end, doMergeBuffers, true);
}
}
public void addOrMerge(long offset, long end, bool doMerge, bool doLogNew)
{
if (doMerge && tail != null && tail.merge(offset, end))
{
return;
}
if (doLogNew)
{
LOG.info("Creating new range; last range (which can include some previous adds) was "
+ tail);
}
DiskRangeList node = new DiskRangeList(offset, end);
if (tail == null)
{
head = tail = node;
}
else
{
tail = tail.insertAfter(node);
}
}
internal static List<DiskRange> getStreamBuffers(DiskRangeList range, long offset, long length)
{
// This assumes sorted ranges (as do many other parts of ORC code.
List<DiskRange> buffers = new List<DiskRange>();
if (length == 0) return buffers;
long streamEnd = offset + length;
bool inRange = false;
while (range != null)
{
if (!inRange)
{
if (range.getEnd() <= offset)
{
range = range.next;
continue; // Skip until we are in range.
}
inRange = true;
if (range.getOffset() < offset)
{
// Partial first buffer, add a slice of it.
buffers.Add(range.sliceAndShift(offset, Math.Min(streamEnd, range.getEnd()), -offset));
if (range.getEnd() >= streamEnd) break; // Partial first buffer is also partial last buffer.
range = range.next;
continue;
}
}
else if (range.getOffset() >= streamEnd)
{
break;
}
if (range.getEnd() > streamEnd)
{
// Partial last buffer (may also be the first buffer), add a slice of it.
buffers.Add(range.sliceAndShift(range.getOffset(), streamEnd, -offset));
break;
}
// Buffer that belongs entirely to one stream.
// TODO: ideally we would want to reuse the object and remove it from the list, but we cannot
// because bufferChunks is also used by clearStreams for zcr. Create a useless dup.
buffers.Add(range.sliceAndShift(range.getOffset(), range.getEnd(), -offset));
if (range.getEnd() == streamEnd) break;
range = range.next;
}
return buffers;
}
/**
* Read the list of ranges from the file.
* @param file the file to read
* @param base the base of the stripe
* @param ranges the disk ranges within the stripe to read
* @return the bytes read for each disk range, which is the same length as
* ranges
* @
*/
static DiskRangeList readDiskRanges(Stream file,
long @base,
DiskRangeList range,
bool doForceDirect)
{
if (range == null) return null;
DiskRangeList prev = range.prev;
if (prev == null)
{
prev = new DiskRangeList.MutateHelper(range);
}
while (range != null)
{
if (range.hasData())
{
range = range.next;
continue;
}
int len = (int)(range.getEnd() - range.getOffset());
long off = range.getOffset();
if (doForceDirect)
{
file.Seek(@base + off, SeekOrigin.Current);
ByteBuffer directBuf = ByteBuffer.allocateDirect(len);
readDirect(file, len, directBuf);
range = range.replaceSelfWith(new RecordReaderImpl.BufferChunk(directBuf, range.getOffset()));
}
else
{
byte[] buffer = new byte[len];
file.readFully((@base + off), buffer, 0, buffer.Length);
range = range.replaceSelfWith(new RecordReaderImpl.BufferChunk(ByteBuffer.wrap(buffer), range.getOffset()));
}
range = range.next;
}
return prev.next;
}
public DiskRangeList readFileData(
DiskRangeList range, long baseOffset, bool doForceDirect)
{
return RecordReaderUtils.readDiskRanges(file, baseOffset, range, doForceDirect);
}
public static void addEntireStreamToRanges(
long offset, long length, DiskRangeList.CreateHelper list, bool doMergeBuffers)
{
list.addOrMerge(offset, offset + length, doMergeBuffers, false);
}
/** Removes an element after current from the list. */
public void removeAfter()
{
DiskRangeList other = this.next;
this.next = other.next;
if (this.next != null)
{
this.next.prev = this;
}
other.next = other.prev = null;
}
/** Replaces this element with another in the list; returns the new element. */
public DiskRangeList replaceSelfWith(DiskRangeList other)
{
other.prev = this.prev;
other.next = this.next;
if (this.prev != null)
{
this.prev.next = other;
}
if (this.next != null)
{
this.next.prev = other;
}
this.next = this.prev = null;
return other;
}
/** Removes the current element from the list. */
public void removeSelf()
{
if (this.prev != null)
{
this.prev.next = this.next;
}
if (this.next != null)
{
this.next.prev = this.prev;
}
this.next = this.prev = null;
}
public void addOrMerge(long offset, long end, bool doMerge, bool doLogNew)
{
if (doMerge && tail != null && tail.merge(offset, end)) return;
if (doLogNew)
{
LOG.info("Creating new range; last range (which can include some previous adds) was "
+ tail);
}
DiskRangeList node = new DiskRangeList(offset, end);
if (tail == null)
{
head = tail = node;
}
else
{
tail = tail.insertAfter(node);
}
}
// @VisibleForTesting
public DiskRangeList[] listToArray()
{
DiskRangeList[] result = new DiskRangeList[listSize()];
int i = 0;
DiskRangeList current = this.next;
while (current != null)
{
result[i] = current;
++i;
current = current.next;
}
return result;
}
public DiskRangeList extract()
{
DiskRangeList result = head;
head = null;
return result;
}
/**
* Inserts an intersecting range after current in the list and adjusts offset accordingly.
* @returns the new element.
*/
public DiskRangeList insertPartAfter(DiskRangeList other)
{
Debug.Assert(other.offset <= this.end);
this.end = other.offset;
return(insertAfter(other));
}
public MutateHelper(DiskRangeList head)
: base(-1, -1)
{
Debug.Assert(head != null);
Debug.Assert(head.prev == null);
this.next = head;
head.prev = this;
}
/**
* Inserts an intersecting range after current in the list and adjusts offset accordingly.
* @returns the new element.
*/
public DiskRangeList insertPartAfter(DiskRangeList other)
{
Debug.Assert(other.offset <= this.end);
this.end = other.offset;
return insertAfter(other);
}
private static DiskRangeList diskRanges(params int[] points)
{
DiskRangeList head = null, tail = null;
for (int i = 0; i < points.Length; i += 2)
{
DiskRangeList range = new DiskRangeList(points[i], points[i + 1]);
if (tail == null)
{
head = tail = range;
}
else
{
tail = tail.insertAfter(range);
}
}
return head;
}
/**
* Inserts an intersecting range before current in the list and adjusts offset accordingly.
* @returns the new element.
*/
public DiskRangeList insertPartBefore(DiskRangeList other)
{
Debug.Assert(other.end >= this.offset);
this.offset = other.end;
other.prev = this.prev;
other.next = this;
if (this.prev != null)
{
this.prev.next = other;
}
this.prev = other;
return other;
}
/**
* Build a string representation of a list of disk ranges.
* @param ranges ranges to stringify
* @return the resulting string
*/
public static string stringifyDiskRanges(DiskRangeList range)
{
StringBuilder buffer = new StringBuilder();
buffer.Append("[");
bool isFirst = true;
while (range != null)
{
if (!isFirst)
{
buffer.Append(", {");
}
else
{
buffer.Append("{");
}
isFirst = false;
buffer.Append(range.ToString());
buffer.Append("}");
range = range.next;
}
buffer.Append("]");
return buffer.ToString();
}
