private PTable(string filename,
Guid id,
int bufferSize = 8096,
int maxReadingThreads = TFConsts.ReadIndexReaderCount,
int depth = 16)
{
if (!File.Exists(filename))
{
throw new CorruptIndexException(new PTableNotFoundException(filename));
}
_id = id;
_size = new FileInfo(filename).Length - PTableHeader.Size - MD5Size;
_filename = filename;
File.SetAttributes(_filename, FileAttributes.ReadOnly);
File.SetAttributes(_filename, FileAttributes.Temporary);
File.SetAttributes(_filename, FileAttributes.NotContentIndexed);
_bufferSize = bufferSize;
_maxReadingThreads = maxReadingThreads;
_buffer = new byte[16];
_bufferPtr = GCHandle.Alloc(_buffer, GCHandleType.Pinned);
using (var stream = File.OpenRead(filename))
using (var reader = new BinaryReader(stream))
{
PTableHeader.FromStream(reader);
}
for (int i = 0; i < _maxReadingThreads; i++)
{
var s = new FileStream(_filename, FileMode.Open, FileAccess.Read, FileShare.Read, 16, FileOptions.RandomAccess);
_streams.Enqueue(s);
}
try
{
_midpoints = PopulateCache(depth);
}
catch (PossibleToHandleOutOfMemoryException)
{
Log.Info("Was unable to create midpoints for ptable. Performance hit possible OOM Exception.");
}
}
private PTable(string filename,
Guid id,
int initialReaders = ESConsts.PTableInitialReaderCount,
int maxReaders = ESConsts.PTableMaxReaderCount,
int depth = 16)
{
Ensure.NotNullOrEmpty(filename, "filename");
Ensure.NotEmptyGuid(id, "id");
Ensure.Positive(maxReaders, "maxReaders");
Ensure.Nonnegative(depth, "depth");
if (!File.Exists(filename))
{
throw new CorruptIndexException(new PTableNotFoundException(filename));
}
_id = id;
_filename = filename;
var sw = Stopwatch.StartNew();
Log.Trace("Loading PTable '{0}' started...", Filename);
_size = new FileInfo(_filename).Length - PTableHeader.Size - MD5Size;
File.SetAttributes(_filename, FileAttributes.ReadOnly | FileAttributes.NotContentIndexed);
_workItems = new ObjectPool <WorkItem>(string.Format("PTable {0} work items", _id),
initialReaders,
maxReaders,
() => new WorkItem(filename, DefaultBufferSize),
workItem => workItem.Dispose(),
pool => OnAllWorkItemsDisposed());
var readerWorkItem = GetWorkItem();
try
{
readerWorkItem.Stream.Seek(0, SeekOrigin.Begin);
var header = PTableHeader.FromStream(readerWorkItem.Stream);
if (header.Version != Version)
{
throw new CorruptIndexException(new WrongFileVersionException(_filename, header.Version, Version));
}
}
catch (Exception)
{
Dispose();
throw;
}
finally
{
ReturnWorkItem(readerWorkItem);
}
try
{
_midpoints = CacheMidpoints(depth);
}
catch (PossibleToHandleOutOfMemoryException)
{
Log.Error("Was unable to create midpoints for PTable '{0}' ({1} entries, depth {2} requested). "
+ "Performance hit possible. OOM Exception.", Filename, Count, depth);
}
Log.Trace("Loading PTable '{0}' ({1} entries, cache depth {2}) done in {3}.", Filename, Count, depth, sw.Elapsed);
}
private PTable(string filename,
Guid id,
int initialReaders,
int maxReaders,
int depth = 16,
bool skipIndexVerify = false)
{
Ensure.NotNullOrEmpty(filename, "filename");
Ensure.NotEmptyGuid(id, "id");
Ensure.Positive(maxReaders, "maxReaders");
Ensure.Nonnegative(depth, "depth");
if (!File.Exists(filename))
{
throw new CorruptIndexException(new PTableNotFoundException(filename));
}
_id = id;
_filename = filename;
Log.Verbose("Loading " + (skipIndexVerify ? "" : "and Verification ") + "of PTable '{pTable}' started...",
Path.GetFileName(Filename));
var sw = Stopwatch.StartNew();
_size = new FileInfo(_filename).Length;
File.SetAttributes(_filename, FileAttributes.ReadOnly | FileAttributes.NotContentIndexed);
_workItems = new ObjectPool <WorkItem>(string.Format("PTable {0} work items", _id),
initialReaders,
maxReaders,
() => new WorkItem(filename, DefaultBufferSize),
workItem => workItem.Dispose(),
pool => OnAllWorkItemsDisposed());
var readerWorkItem = GetWorkItem();
try {
readerWorkItem.Stream.Seek(0, SeekOrigin.Begin);
var header = PTableHeader.FromStream(readerWorkItem.Stream);
if ((header.Version != PTableVersions.IndexV1) &&
(header.Version != PTableVersions.IndexV2) &&
(header.Version != PTableVersions.IndexV3) &&
(header.Version != PTableVersions.IndexV4))
{
throw new CorruptIndexException(new WrongFileVersionException(_filename, header.Version, Version));
}
_version = header.Version;
if (_version == PTableVersions.IndexV1)
{
_indexEntrySize = IndexEntryV1Size;
_indexKeySize = IndexKeyV1Size;
}
if (_version == PTableVersions.IndexV2)
{
_indexEntrySize = IndexEntryV2Size;
_indexKeySize = IndexKeyV2Size;
}
if (_version == PTableVersions.IndexV3)
{
_indexEntrySize = IndexEntryV3Size;
_indexKeySize = IndexKeyV3Size;
}
if (_version >= PTableVersions.IndexV4)
{
//read the PTable footer
var previousPosition = readerWorkItem.Stream.Position;
readerWorkItem.Stream.Seek(readerWorkItem.Stream.Length - MD5Size - PTableFooter.GetSize(_version),
SeekOrigin.Begin);
var footer = PTableFooter.FromStream(readerWorkItem.Stream);
if (footer.Version != header.Version)
{
throw new CorruptIndexException(
String.Format("PTable header/footer version mismatch: {0}/{1}", header.Version,
footer.Version), new InvalidFileException("Invalid PTable file."));
}
if (_version == PTableVersions.IndexV4)
{
_indexEntrySize = IndexEntryV4Size;
_indexKeySize = IndexKeyV4Size;
}
else
{
throw new InvalidOperationException("Unknown PTable version: " + _version);
}
_midpointsCached = footer.NumMidpointsCached;
_midpointsCacheSize = _midpointsCached * _indexEntrySize;
readerWorkItem.Stream.Seek(previousPosition, SeekOrigin.Begin);
}
long indexEntriesTotalSize = (_size - PTableHeader.Size - _midpointsCacheSize -
PTableFooter.GetSize(_version) - MD5Size);
if (indexEntriesTotalSize < 0)
{
throw new CorruptIndexException(String.Format(
"Total size of index entries < 0: {0}. _size: {1}, header size: {2}, _midpointsCacheSize: {3}, footer size: {4}, md5 size: {5}",
indexEntriesTotalSize, _size, PTableHeader.Size, _midpointsCacheSize,
PTableFooter.GetSize(_version), MD5Size));
}
else if (indexEntriesTotalSize % _indexEntrySize != 0)
{
throw new CorruptIndexException(String.Format(
"Total size of index entries: {0} is not divisible by index entry size: {1}",
indexEntriesTotalSize, _indexEntrySize));
}
_count = indexEntriesTotalSize / _indexEntrySize;
if (_version >= PTableVersions.IndexV4 && _count > 0 && _midpointsCached > 0 && _midpointsCached < 2)
{
//if there is at least 1 index entry with version>=4 and there are cached midpoints, there should always be at least 2 midpoints cached
throw new CorruptIndexException(String.Format(
"Less than 2 midpoints cached in PTable. Index entries: {0}, Midpoints cached: {1}", _count,
_midpointsCached));
}
else if (_count >= 2 && _midpointsCached > _count)
{
//if there are at least 2 index entries, midpoints count should be at most the number of index entries
throw new CorruptIndexException(String.Format(
"More midpoints cached in PTable than index entries. Midpoints: {0} , Index entries: {1}",
_midpointsCached, _count));
}
if (Count == 0)
{
_minEntry = new IndexEntryKey(ulong.MaxValue, long.MaxValue);
_maxEntry = new IndexEntryKey(ulong.MinValue, long.MinValue);
}
else
{
var minEntry = ReadEntry(_indexEntrySize, Count - 1, readerWorkItem, _version);
_minEntry = new IndexEntryKey(minEntry.Stream, minEntry.Version);
var maxEntry = ReadEntry(_indexEntrySize, 0, readerWorkItem, _version);
_maxEntry = new IndexEntryKey(maxEntry.Stream, maxEntry.Version);
}
} catch (Exception) {
Dispose();
throw;
} finally {
ReturnWorkItem(readerWorkItem);
}
int calcdepth = 0;
try {
calcdepth = GetDepth(_count * _indexEntrySize, depth);
_midpoints = CacheMidpointsAndVerifyHash(calcdepth, skipIndexVerify);
} catch (PossibleToHandleOutOfMemoryException) {
Log.Error(
"Unable to create midpoints for PTable '{pTable}' ({count} entries, depth {depth} requested). "
+ "Performance hit will occur. OOM Exception.", Path.GetFileName(Filename), Count, depth);
}
Log.Verbose(
"Loading PTable (Version: {version}) '{pTable}' ({count} entries, cache depth {depth}) done in {elapsed}.",
_version, Path.GetFileName(Filename), Count, calcdepth, sw.Elapsed);
}
private PTable(string filename,
Guid id,
int initialReaders = ESConsts.PTableInitialReaderCount,
int maxReaders = ESConsts.PTableMaxReaderCount,
int depth = 16)
{
Ensure.NotNullOrEmpty(filename, "filename");
Ensure.NotEmptyGuid(id, "id");
Ensure.Positive(maxReaders, "maxReaders");
Ensure.Nonnegative(depth, "depth");
if (!File.Exists(filename))
{
throw new CorruptIndexException(new PTableNotFoundException(filename));
}
_id = id;
_filename = filename;
Log.Trace("Loading and Verification of PTable '{0}' started...", Path.GetFileName(Filename));
var sw = Stopwatch.StartNew();
_size = new FileInfo(_filename).Length;
File.SetAttributes(_filename, FileAttributes.ReadOnly | FileAttributes.NotContentIndexed);
_workItems = new ObjectPool <WorkItem>(string.Format("PTable {0} work items", _id),
initialReaders,
maxReaders,
() => new WorkItem(filename, DefaultBufferSize),
workItem => workItem.Dispose(),
pool => OnAllWorkItemsDisposed());
var readerWorkItem = GetWorkItem();
try
{
readerWorkItem.Stream.Seek(0, SeekOrigin.Begin);
var header = PTableHeader.FromStream(readerWorkItem.Stream);
if ((header.Version != PTableVersions.Index32Bit) &&
(header.Version != PTableVersions.Index64Bit))
{
throw new CorruptIndexException(new WrongFileVersionException(_filename, header.Version, Version));
}
_version = header.Version;
if (_version == PTableVersions.Index32Bit)
{
_indexEntrySize = IndexEntry32Size;
_indexKeySize = IndexKey32Size;
}
if (_version == PTableVersions.Index64Bit)
{
_indexEntrySize = IndexEntry64Size;
_indexKeySize = IndexKey64Size;
}
_count = ((_size - PTableHeader.Size - MD5Size) / _indexEntrySize);
if (Count == 0)
{
_minEntry = new IndexEntryKey(ulong.MaxValue, int.MaxValue);
_maxEntry = new IndexEntryKey(ulong.MinValue, int.MinValue);
}
else
{
var minEntry = ReadEntry(_indexEntrySize, Count - 1, readerWorkItem, _version);
_minEntry = new IndexEntryKey(minEntry.Stream, minEntry.Version);
var maxEntry = ReadEntry(_indexEntrySize, 0, readerWorkItem, _version);
_maxEntry = new IndexEntryKey(maxEntry.Stream, maxEntry.Version);
}
}
catch (Exception)
{
Dispose();
throw;
}
finally
{
ReturnWorkItem(readerWorkItem);
}
int calcdepth = 0;
try
{
calcdepth = GetDepth(_size, depth);
_midpoints = CacheMidpointsAndVerifyHash(calcdepth);
}
catch (PossibleToHandleOutOfMemoryException)
{
Log.Error("Unable to create midpoints for PTable '{0}' ({1} entries, depth {2} requested). "
+ "Performance hit will occur. OOM Exception.", Path.GetFileName(Filename), Count, depth);
}
Log.Trace("Loading PTable (Version: {0}) '{1}' ({2} entries, cache depth {3}) done in {4}.",
_version, Path.GetFileName(Filename), Count, calcdepth, sw.Elapsed);
}
