private Tuple<int, int> TranslateClosestForwardWithoutMidpoints(ReaderWorkItem workItem, int pos, int startIndex, int endIndex) { PosMap res = ReadPosMap(workItem, endIndex); if (pos > res.LogPos) return Tuple.Create(Chunk.ActualDataSize, endIndex + 1); // to allow backward reading of the last record, forward read will decline anyway int low = startIndex; int high = endIndex; while (low < high) { var mid = low + (high - low) / 2; var v = ReadPosMap(workItem, mid); if (v.LogPos < pos) low = mid + 1; else { high = mid; res = v; } } return Tuple.Create(res.ActualPos, high); }
private Tuple<int, int> TranslateClosestForwardPosition(ReaderWorkItem workItem, int logicalPosition) { if (!Chunk.IsReadOnly || Chunk.ChunkFooter.MapSize == 0) { // this is mostly for ability to read closest backward from the very end var logicalPos = Math.Min(Chunk.ActualDataSize, logicalPosition); return Tuple.Create(logicalPos, -1); } var midpoints = _midpoints; if (workItem.IsMemory || midpoints == null) { return TranslateClosestForwardWithoutMidpoints(workItem, logicalPosition, 0, Chunk.ChunkFooter.MapCount - 1); } return TranslateClosestForwardWithMidpoints(workItem, midpoints, logicalPosition); }
private Tuple<int, int> TranslateClosestForwardWithMidpoints(ReaderWorkItem workItem, Midpoint[] midpoints, int pos) { if (pos > midpoints[midpoints.Length - 1].LogPos) return Tuple.Create(Chunk.ActualDataSize, midpoints.Length); // to allow backward reading of the last record, forward read will decline anyway var recordRange = LocatePosRange(midpoints, pos); return TranslateClosestForwardWithoutMidpoints(workItem, pos, recordRange.Item1, recordRange.Item2); }
private int TranslateExactWithoutMidpoints(ReaderWorkItem workItem, int pos, int startIndex, int endIndex) { int low = startIndex; int high = endIndex; while (low <= high) { var mid = low + (high - low) / 2; var v = ReadPosMap(workItem, mid); if (v.LogPos == pos) return v.ActualPos; if (v.LogPos < pos) low = mid + 1; else high = mid - 1; } return -1; }
private int TranslateExactWithMidpoints(ReaderWorkItem workItem, Midpoint[] midpoints, int pos) { if (pos < midpoints[0].LogPos || pos > midpoints[midpoints.Length - 1].LogPos) return -1; var recordRange = LocatePosRange(midpoints, pos); return TranslateExactWithoutMidpoints(workItem, pos, recordRange.Item1, recordRange.Item2); }
protected bool TryReadForwardInternal(ReaderWorkItem workItem, long actualPosition, out int length, out LogRecord record) { length = -1; record = null; workItem.Stream.Position = GetRawPosition(actualPosition); if (actualPosition + 2*sizeof(int) > Chunk.PhysicalDataSize) // no space even for length prefix and suffix return false; length = workItem.Reader.ReadInt32(); if (length <= 0) { throw new InvalidReadException( string.Format("Log record at actual pos {0} has non-positive length: {1}. " + " in chunk.", actualPosition, length, Chunk)); } if (length > TFConsts.MaxLogRecordSize) { throw new InvalidReadException( string.Format("Log record at actual pos {0} has too large length: {1} bytes, " + "while limit is {2} bytes. In chunk {3}.", actualPosition, length, TFConsts.MaxLogRecordSize, Chunk)); } if (actualPosition + length + 2 * sizeof(int) > Chunk.PhysicalDataSize) { throw new UnableToReadPastEndOfStreamException( string.Format("There is not enough space to read full record (length prefix: {0}). " + "Actual pre-position: {1}. Something is seriously wrong in chunk {2}.", length, actualPosition, Chunk)); } record = LogRecord.ReadFrom(workItem.Reader); // verify suffix length == prefix length int suffixLength = workItem.Reader.ReadInt32(); if (suffixLength != length) { throw new Exception( string.Format("Prefix/suffix length inconsistency: prefix length({0}) != suffix length ({1}).\n" + "Actual pre-position: {2}. Something is seriously wrong in chunk {3}.", length, suffixLength, actualPosition, Chunk)); } return true; }
private int TranslateExactPosition(ReaderWorkItem workItem, int pos) { var midpoints = _midpoints; if (workItem.IsMemory || midpoints == null) return TranslateExactWithoutMidpoints(workItem, pos, 0, Chunk.ChunkFooter.MapCount - 1); return TranslateExactWithMidpoints(workItem, midpoints, pos); }
private int TranslateExactPosition(ReaderWorkItem workItem, long pos) { var midpoints = _midpoints; if (workItem.IsMemory || midpoints == null) return TranslateExactWithoutMidpoints(workItem, pos, 0, Chunk.ChunkFooter.MapCount - 1); return TranslateExactWithMidpoints(workItem, midpoints, pos); }
private void ReturnReaderWorkItem(ReaderWorkItem item) { if (item.IsMemory) { _memStreams.Enqueue(item); if (_isCached == 0 || _selfdestructin54321) TryDestructMemStreams(); } else { _fileStreams.Enqueue(item); if (_selfdestructin54321) TryDestructFileStreams(); } }
private int TranslateClosestForwardWithMidpoints(ReaderWorkItem workItem, Midpoint[] midpoints, long pos) { // to allow backward reading of the last record, forward read will decline anyway if (pos > midpoints[midpoints.Length - 1].LogPos) return Chunk.PhysicalDataSize; var recordRange = LocatePosRange(midpoints, pos); return TranslateClosestForwardWithoutMidpoints(workItem, pos, recordRange.Lower, recordRange.Upper); }
protected bool TryReadBackwardInternal(ReaderWorkItem workItem, long actualPosition, out int length, out LogRecord record) { length = -1; record = null; if (actualPosition < 2 * sizeof(int)) // no space even for length prefix and suffix return false; var realPos = GetRawPosition(actualPosition); workItem.Stream.Position = realPos - sizeof(int); length = workItem.Reader.ReadInt32(); if (length <= 0) { throw new ArgumentException( string.Format("Log record that ends at actual pos {0} has non-positive length: {1}. " + "Something is seriously wrong in chunk {2}.", actualPosition, length, Chunk)); } if (length > TFConsts.MaxLogRecordSize) { throw new ArgumentException( string.Format("Log record that ends at actual pos {0} has too large length: {1} bytes, " + "while limit is {2} bytes. Something is seriously wrong in chunk {3}.", actualPosition, length, TFConsts.MaxLogRecordSize, Chunk)); } if (actualPosition < length + 2 * sizeof(int)) // no space for record + length prefix and suffix { throw new UnableToReadPastEndOfStreamException( string.Format("There is not enough space to read full record (length suffix: {0}). " + "Actual post-position: {1}. Something is seriously wrong in chunk {2}.", length, actualPosition, Chunk)); } workItem.Stream.Position = realPos - length - 2*sizeof(int); // verify suffix length == prefix length int prefixLength = workItem.Reader.ReadInt32(); if (prefixLength != length) { throw new Exception( string.Format("Prefix/suffix length inconsistency: prefix length({0}) != suffix length ({1})" + "Actual post-position: {2}. Something is seriously wrong in chunk {3}.", prefixLength, length, actualPosition, Chunk)); } record = LogRecord.ReadFrom(workItem.Reader); return true; }
private int TranslateClosestForwardPosition(ReaderWorkItem workItem, long logicalPosition) { var midpoints = _midpoints; if (workItem.IsMemory || midpoints == null) return TranslateClosestForwardWithoutMidpoints(workItem, logicalPosition, 0, Chunk.ChunkFooter.MapCount - 1); return TranslateClosestForwardWithMidpoints(workItem, midpoints, logicalPosition); }
private PosMap ReadPosMap(ReaderWorkItem workItem, int index) { if (Chunk.ChunkFooter.IsMap12Bytes) { var pos = ChunkHeader.Size + Chunk.ChunkFooter.PhysicalDataSize + index*PosMap.FullSize; workItem.Stream.Seek(pos, SeekOrigin.Begin); return PosMap.FromNewFormat(workItem.Reader); } else { var pos = ChunkHeader.Size + Chunk.ChunkFooter.PhysicalDataSize + index*PosMap.DeprecatedSize; workItem.Stream.Seek(pos, SeekOrigin.Begin); return PosMap.FromOldFormat(workItem.Reader); } }
protected bool TryReadBackwardInternal(ReaderWorkItem workItem, long actualPosition, out int length, out LogRecord record) { length = -1; record = null; // no space even for length prefix and suffix if (actualPosition < 2 * sizeof(int)) { _log.Warning( "Tried to read actual position {actualPosition}, but the position isn't large enough to contain a record", actualPosition); return(false); } var realPos = GetRawPosition(actualPosition); workItem.Stream.Position = realPos - sizeof(int); length = workItem.Reader.ReadInt32(); if (length <= 0) { throw new InvalidReadException( string.Format("Log record that ends at actual pos {0} has non-positive length: {1}. " + "In chunk {2}.", actualPosition, length, Chunk)); } if (length > TFConsts.MaxLogRecordSize) { throw new ArgumentException( string.Format("Log record that ends at actual pos {0} has too large length: {1} bytes, " + "while limit is {2} bytes. In chunk {3}.", actualPosition, length, TFConsts.MaxLogRecordSize, Chunk)); } if (actualPosition < length + 2 * sizeof(int)) // no space for record + length prefix and suffix { throw new UnableToReadPastEndOfStreamException( string.Format("There is not enough space to read full record (length suffix: {0}). " + "Actual post-position: {1}. Something is seriously wrong in chunk {2}.", length, actualPosition, Chunk)); } workItem.Stream.Position = realPos - length - 2 * sizeof(int); // verify suffix length == prefix length int prefixLength = workItem.Reader.ReadInt32(); if (prefixLength != length) { throw new Exception( string.Format("Prefix/suffix length inconsistency: prefix length({0}) != suffix length ({1})" + "Actual post-position: {2}. Something is seriously wrong in chunk {3}.", prefixLength, length, actualPosition, Chunk)); } record = LogRecord.ReadFrom(workItem.Reader); return(true); }
protected bool TryReadForwardInternal(ReaderWorkItem workItem, int actualPosition, out int length, out LogRecord record) { length = -1; record = null; workItem.Stream.Position = GetRealPosition(actualPosition); if (actualPosition + 2*sizeof(int) > Chunk.ActualDataSize) // no space even for length prefix and suffix return false; length = workItem.Reader.ReadInt32(); if (length <= 0) { throw new ArgumentException( string.Format("Log record at actual pos {0} has non-positive length: {1}. " + "Something is seriously wrong in chunk {2}-{3} ({4}).", actualPosition, length, Chunk.ChunkHeader.ChunkStartNumber, Chunk.ChunkHeader.ChunkEndNumber, Chunk.FileName)); } if (length > TFConsts.MaxLogRecordSize) { throw new ArgumentException( string.Format("Log record at actual pos {0} has too large length: {1} bytes, " + "while limit is {2} bytes. Something is seriously wrong in chunk {3}-{4} ({5}).", actualPosition, length, TFConsts.MaxLogRecordSize, Chunk.ChunkHeader.ChunkStartNumber, Chunk.ChunkHeader.ChunkEndNumber, Chunk.FileName)); } if (actualPosition + length + 2*sizeof(int) > Chunk.ActualDataSize) throw new UnableToReadPastEndOfStreamException( string.Format("There is not enough space to read full record (record length according to length prefix: {0}).", length)); record = LogRecord.ReadFrom(workItem.Reader); #if DEBUG // verify suffix length == prefix length int suffixLength = workItem.Reader.ReadInt32(); Debug.Assert(suffixLength == length, "Prefix/suffix length inconsistency ", "Prefix length({0}) != suffix length ({1})", length, suffixLength); #endif return true; }
private PosMap ReadPosMap(ReaderWorkItem workItem, int index) { var pos = ChunkHeader.Size + Chunk.ChunkFooter.ActualChunkSize + index * PosMap.Size; workItem.Stream.Seek(pos, SeekOrigin.Begin); return new PosMap(workItem.Reader.ReadUInt64()); }
protected static bool TryReadBackwardInternal(ReaderWorkItem workItem, int actualPosition, out int length, out LogRecord record) { length = -1; record = null; if (actualPosition < 2 * sizeof(int)) // no space even for length prefix and suffix return false; var realPos = GetRealPosition(actualPosition); workItem.Stream.Position = realPos - sizeof(int); length = workItem.Reader.ReadInt32(); if (length <= 0) { throw new ArgumentException( string.Format("Log record that ends at actual pos {0} has non-positive length: {1}. " + "Something is seriously wrong.", actualPosition, length)); } if (length > TFConsts.MaxLogRecordSize) { throw new ArgumentException( string.Format("Log record that ends at actual pos {0} has too large length: {1} bytes, " + "while limit is {2} bytes.", actualPosition, length, TFConsts.MaxLogRecordSize)); } if (actualPosition < length + 2 * sizeof(int)) // no space for record + length prefix and suffix throw new UnableToReadPastEndOfStreamException( string.Format("There is not enough space to read full record (record length according to length suffix: {0}).", length)); workItem.Stream.Position = realPos - length - sizeof(int); record = LogRecord.ReadFrom(workItem.Reader); #if DEBUG workItem.Stream.Position = realPos - length - 2 * sizeof(int); var prefixLength = workItem.Reader.ReadInt32(); Debug.Assert(prefixLength == length); #endif return true; }
private int TranslateClosestForwardWithoutMidpoints(ReaderWorkItem workItem, long pos, long startIndex, long endIndex) { PosMap res = ReadPosMap(workItem, endIndex); // to allow backward reading of the last record, forward read will decline anyway if (pos > res.LogPos) return Chunk.PhysicalDataSize; long low = startIndex; long high = endIndex; while (low < high) { var mid = low + (high - low) / 2; var v = ReadPosMap(workItem, mid); if (v.LogPos < pos) low = mid + 1; else { high = mid; res = v; } } return res.ActualPos; }