private static void WriteMidpointsTo(BufferedStream bs, FileStream fs, byte version, int indexEntrySize, byte[] buffer, long dumpedEntryCount, long numIndexEntries, long requiredMidpointCount, List <Midpoint> midpoints)
{
//WRITE MIDPOINT ENTRIES
//special case, when there is a single index entry, we need two midpoints
if (numIndexEntries == 1 && midpoints.Count == 1)
{
midpoints.Add(new Midpoint(midpoints[0].Key, midpoints[0].ItemIndex));
}
var midpointsWritten = 0;
if (dumpedEntryCount == numIndexEntries && requiredMidpointCount == midpoints.Count) //if these values don't match, something is wrong
{
bs.Flush();
fs.SetLength(fs.Position + midpoints.Count * indexEntrySize);
foreach (var pt in midpoints)
{
AppendMidpointRecordTo(bs, buffer, version, pt, indexEntrySize);
}
midpointsWritten = midpoints.Count;
Log.Debug("Cached {midpointsWritten} index midpoints to PTable", midpointsWritten);
}
else
{
Log.Debug("Not caching index midpoints to PTable due to count mismatch. Table entries: {numIndexEntries} / Dumped entries: {dumpedEntryCount}, Required midpoint count: {requiredMidpointCount} / Actual midpoint count: {midpoints}", numIndexEntries, dumpedEntryCount, requiredMidpointCount, midpoints.Count);
}
bs.Flush();
fs.SetLength(fs.Position + PTableFooter.GetSize(version));
var footerBytes = new PTableFooter(version, (uint)midpointsWritten).AsByteArray();
bs.Write(footerBytes, 0, footerBytes.Length);
bs.Flush();
}
private PTable(string filename,
Guid id,
int initialReaders,
int maxReaders,
int depth = 16,
bool skipIndexVerify = false)
{
Ensure.NotNullOrEmpty(filename, "filename");
Ensure.NotEmptyGuid(id, "id");
Ensure.Positive(maxReaders, "maxReaders");
Ensure.Nonnegative(depth, "depth");
if (!File.Exists(filename))
{
throw new CorruptIndexException(new PTableNotFoundException(filename));
}
_id = id;
_filename = filename;
Log.Verbose("Loading " + (skipIndexVerify ? "" : "and Verification ") + "of PTable '{pTable}' started...",
Path.GetFileName(Filename));
var sw = Stopwatch.StartNew();
_size = new FileInfo(_filename).Length;
File.SetAttributes(_filename, FileAttributes.ReadOnly | FileAttributes.NotContentIndexed);
_workItems = new ObjectPool <WorkItem>(string.Format("PTable {0} work items", _id),
initialReaders,
maxReaders,
() => new WorkItem(filename, DefaultBufferSize),
workItem => workItem.Dispose(),
pool => OnAllWorkItemsDisposed());
var readerWorkItem = GetWorkItem();
try {
readerWorkItem.Stream.Seek(0, SeekOrigin.Begin);
var header = PTableHeader.FromStream(readerWorkItem.Stream);
if ((header.Version != PTableVersions.IndexV1) &&
(header.Version != PTableVersions.IndexV2) &&
(header.Version != PTableVersions.IndexV3) &&
(header.Version != PTableVersions.IndexV4))
{
throw new CorruptIndexException(new WrongFileVersionException(_filename, header.Version, Version));
}
_version = header.Version;
if (_version == PTableVersions.IndexV1)
{
_indexEntrySize = IndexEntryV1Size;
_indexKeySize = IndexKeyV1Size;
}
if (_version == PTableVersions.IndexV2)
{
_indexEntrySize = IndexEntryV2Size;
_indexKeySize = IndexKeyV2Size;
}
if (_version == PTableVersions.IndexV3)
{
_indexEntrySize = IndexEntryV3Size;
_indexKeySize = IndexKeyV3Size;
}
if (_version >= PTableVersions.IndexV4)
{
//read the PTable footer
var previousPosition = readerWorkItem.Stream.Position;
readerWorkItem.Stream.Seek(readerWorkItem.Stream.Length - MD5Size - PTableFooter.GetSize(_version),
SeekOrigin.Begin);
var footer = PTableFooter.FromStream(readerWorkItem.Stream);
if (footer.Version != header.Version)
{
throw new CorruptIndexException(
String.Format("PTable header/footer version mismatch: {0}/{1}", header.Version,
footer.Version), new InvalidFileException("Invalid PTable file."));
}
if (_version == PTableVersions.IndexV4)
{
_indexEntrySize = IndexEntryV4Size;
_indexKeySize = IndexKeyV4Size;
}
else
{
throw new InvalidOperationException("Unknown PTable version: " + _version);
}
_midpointsCached = footer.NumMidpointsCached;
_midpointsCacheSize = _midpointsCached * _indexEntrySize;
readerWorkItem.Stream.Seek(previousPosition, SeekOrigin.Begin);
}
long indexEntriesTotalSize = (_size - PTableHeader.Size - _midpointsCacheSize -
PTableFooter.GetSize(_version) - MD5Size);
if (indexEntriesTotalSize < 0)
{
throw new CorruptIndexException(String.Format(
"Total size of index entries < 0: {0}. _size: {1}, header size: {2}, _midpointsCacheSize: {3}, footer size: {4}, md5 size: {5}",
indexEntriesTotalSize, _size, PTableHeader.Size, _midpointsCacheSize,
PTableFooter.GetSize(_version), MD5Size));
}
else if (indexEntriesTotalSize % _indexEntrySize != 0)
{
throw new CorruptIndexException(String.Format(
"Total size of index entries: {0} is not divisible by index entry size: {1}",
indexEntriesTotalSize, _indexEntrySize));
}
_count = indexEntriesTotalSize / _indexEntrySize;
if (_version >= PTableVersions.IndexV4 && _count > 0 && _midpointsCached > 0 && _midpointsCached < 2)
{
//if there is at least 1 index entry with version>=4 and there are cached midpoints, there should always be at least 2 midpoints cached
throw new CorruptIndexException(String.Format(
"Less than 2 midpoints cached in PTable. Index entries: {0}, Midpoints cached: {1}", _count,
_midpointsCached));
}
else if (_count >= 2 && _midpointsCached > _count)
{
//if there are at least 2 index entries, midpoints count should be at most the number of index entries
throw new CorruptIndexException(String.Format(
"More midpoints cached in PTable than index entries. Midpoints: {0} , Index entries: {1}",
_midpointsCached, _count));
}
if (Count == 0)
{
_minEntry = new IndexEntryKey(ulong.MaxValue, long.MaxValue);
_maxEntry = new IndexEntryKey(ulong.MinValue, long.MinValue);
}
else
{
var minEntry = ReadEntry(_indexEntrySize, Count - 1, readerWorkItem, _version);
_minEntry = new IndexEntryKey(minEntry.Stream, minEntry.Version);
var maxEntry = ReadEntry(_indexEntrySize, 0, readerWorkItem, _version);
_maxEntry = new IndexEntryKey(maxEntry.Stream, maxEntry.Version);
}
} catch (Exception) {
Dispose();
throw;
} finally {
ReturnWorkItem(readerWorkItem);
}
int calcdepth = 0;
try {
calcdepth = GetDepth(_count * _indexEntrySize, depth);
_midpoints = CacheMidpointsAndVerifyHash(calcdepth, skipIndexVerify);
} catch (PossibleToHandleOutOfMemoryException) {
Log.Error(
"Unable to create midpoints for PTable '{pTable}' ({count} entries, depth {depth} requested). "
+ "Performance hit will occur. OOM Exception.", Path.GetFileName(Filename), Count, depth);
}
Log.Verbose(
"Loading PTable (Version: {version}) '{pTable}' ({count} entries, cache depth {depth}) done in {elapsed}.",
_version, Path.GetFileName(Filename), Count, calcdepth, sw.Elapsed);
}
internal Midpoint[] CacheMidpointsAndVerifyHash(int depth, bool skipIndexVerify)
{
var buffer = new byte[4096];
if (depth < 0 || depth > 30)
{
throw new ArgumentOutOfRangeException("depth");
}
var count = Count;
if (count == 0 || depth == 0)
{
return(null);
}
if (skipIndexVerify)
{
Log.Debug("Disabling Verification of PTable");
}
Stream stream = null;
WorkItem workItem = null;
if (Runtime.IsUnixOrMac)
{
workItem = GetWorkItem();
stream = workItem.Stream;
}
else
{
stream = UnbufferedFileStream.Create(_filename, FileMode.Open, FileAccess.Read, FileShare.Read, false,
4096, 4096, false, 4096);
}
try {
int midpointsCount;
Midpoint[] midpoints;
using (MD5 md5 = MD5.Create()) {
try {
midpointsCount = (int)Math.Max(2L, Math.Min((long)1 << depth, count));
midpoints = new Midpoint[midpointsCount];
} catch (OutOfMemoryException exc) {
throw new PossibleToHandleOutOfMemoryException("Failed to allocate memory for Midpoint cache.",
exc);
}
if (skipIndexVerify && (_version >= PTableVersions.IndexV4))
{
if (_midpointsCached == midpointsCount)
{
//index verification is disabled and cached midpoints with the same depth requested are available
//so, we can load them directly from the PTable file
Log.Debug("Loading {midpointsCached} cached midpoints from PTable", _midpointsCached);
long startOffset = stream.Length - MD5Size - PTableFooter.GetSize(_version) -
_midpointsCacheSize;
stream.Seek(startOffset, SeekOrigin.Begin);
for (uint k = 0; k < _midpointsCached; k++)
{
stream.Read(buffer, 0, _indexEntrySize);
IndexEntryKey key;
long index;
if (_version == PTableVersions.IndexV4)
{
key = new IndexEntryKey(BitConverter.ToUInt64(buffer, 8),
BitConverter.ToInt64(buffer, 0));
index = BitConverter.ToInt64(buffer, 8 + 8);
}
else
{
throw new InvalidOperationException("Unknown PTable version: " + _version);
}
midpoints[k] = new Midpoint(key, index);
if (k > 0)
{
if (midpoints[k].Key.GreaterThan(midpoints[k - 1].Key))
{
throw new CorruptIndexException(String.Format(
"Index entry key for midpoint {0} (stream: {1}, version: {2}) < index entry key for midpoint {3} (stream: {4}, version: {5})",
k - 1, midpoints[k - 1].Key.Stream, midpoints[k - 1].Key.Version, k,
midpoints[k].Key.Stream, midpoints[k].Key.Version));
}
else if (midpoints[k - 1].ItemIndex > midpoints[k].ItemIndex)
{
throw new CorruptIndexException(String.Format(
"Item index for midpoint {0} ({1}) > Item index for midpoint {2} ({3})",
k - 1, midpoints[k - 1].ItemIndex, k, midpoints[k].ItemIndex));
}
}
}
return(midpoints);
}
else
{
Log.Debug(
"Skipping loading of cached midpoints from PTable due to count mismatch, cached midpoints: {midpointsCached} / required midpoints: {midpointsCount}",
_midpointsCached, midpointsCount);
}
}
if (!skipIndexVerify)
{
stream.Seek(0, SeekOrigin.Begin);
stream.Read(buffer, 0, PTableHeader.Size);
md5.TransformBlock(buffer, 0, PTableHeader.Size, null, 0);
}
long previousNextIndex = long.MinValue;
var previousKey = new IndexEntryKey(long.MaxValue, long.MaxValue);
for (long k = 0; k < midpointsCount; ++k)
{
long nextIndex = GetMidpointIndex(k, count, midpointsCount);
if (previousNextIndex != nextIndex)
{
if (!skipIndexVerify)
{
ReadUntilWithMd5(PTableHeader.Size + _indexEntrySize * nextIndex, stream, md5);
stream.Read(buffer, 0, _indexKeySize);
md5.TransformBlock(buffer, 0, _indexKeySize, null, 0);
}
else
{
stream.Seek(PTableHeader.Size + _indexEntrySize * nextIndex, SeekOrigin.Begin);
stream.Read(buffer, 0, _indexKeySize);
}
IndexEntryKey key;
if (_version == PTableVersions.IndexV1)
{
key = new IndexEntryKey(BitConverter.ToUInt32(buffer, 4),
BitConverter.ToInt32(buffer, 0));
}
else if (_version == PTableVersions.IndexV2)
{
key = new IndexEntryKey(BitConverter.ToUInt64(buffer, 4),
BitConverter.ToInt32(buffer, 0));
}
else
{
key = new IndexEntryKey(BitConverter.ToUInt64(buffer, 8),
BitConverter.ToInt64(buffer, 0));
}
midpoints[k] = new Midpoint(key, nextIndex);
previousNextIndex = nextIndex;
previousKey = key;
}
else
{
midpoints[k] = new Midpoint(previousKey, previousNextIndex);
}
if (k > 0)
{
if (midpoints[k].Key.GreaterThan(midpoints[k - 1].Key))
{
throw new CorruptIndexException(String.Format(
"Index entry key for midpoint {0} (stream: {1}, version: {2}) < index entry key for midpoint {3} (stream: {4}, version: {5})",
k - 1, midpoints[k - 1].Key.Stream, midpoints[k - 1].Key.Version, k,
midpoints[k].Key.Stream, midpoints[k].Key.Version));
}
else if (midpoints[k - 1].ItemIndex > midpoints[k].ItemIndex)
{
throw new CorruptIndexException(String.Format(
"Item index for midpoint {0} ({1}) > Item index for midpoint {2} ({3})", k - 1,
midpoints[k - 1].ItemIndex, k, midpoints[k].ItemIndex));
}
}
}
if (!skipIndexVerify)
{
ReadUntilWithMd5(stream.Length - MD5Size, stream, md5);
//verify hash (should be at stream.length - MD5Size)
md5.TransformFinalBlock(Empty.ByteArray, 0, 0);
var fileHash = new byte[MD5Size];
stream.Read(fileHash, 0, MD5Size);
ValidateHash(md5.Hash, fileHash);
}
return(midpoints);
}
} catch {
Dispose();
throw;
} finally {
if (Runtime.IsUnixOrMac)
{
if (workItem != null)
{
ReturnWorkItem(workItem);
}
}
else
{
if (stream != null)
{
stream.Dispose();
}
}
}
}
