private static List <List <PTable> > LoadPTables(StreamReader reader, string indexmapFilename, TFPos checkpoints, int cacheDepth, bool skipIndexVerify,
int threads)
{
var tables = new List <List <PTable> >();
try
{
try
{
Parallel.ForEach(GetAllLines(reader).Reverse(), // Reverse so we load the highest levels (biggest files) first - ensures we use concurrency in the most efficient way.
new ParallelOptions {
MaxDegreeOfParallelism = threads
},
indexMapEntry =>
{
if (checkpoints.PreparePosition < 0 || checkpoints.CommitPosition < 0)
{
throw new CorruptIndexException(
string.Format("Negative prepare/commit checkpoint in non-empty IndexMap: {0}.", checkpoints));
}
PTable ptable = null;
var pieces = indexMapEntry.Split(',');
try
{
var level = int.Parse(pieces[0]);
var position = int.Parse(pieces[1]);
var file = pieces[2];
var path = Path.GetDirectoryName(indexmapFilename);
var ptablePath = Path.Combine(path, file);
ptable = PTable.FromFile(ptablePath, cacheDepth, skipIndexVerify);
lock (tables)
{
InsertTableToTables(tables, level, position, ptable);
}
}
catch (Exception)
{
// if PTable file path was correct, but data is corrupted, we still need to dispose opened streams
if (ptable != null)
{
ptable.Dispose();
}
throw;
}
});
// Verify map is correct
for (int i = 0; i < tables.Count; ++i)
{
for (int j = 0; j < tables[i].Count; ++j)
{
if (tables[i][j] == null)
{
throw new CorruptIndexException($"indexmap is missing contiguous level,position {i},{j}");
}
}
}
}
catch (AggregateException aggEx)
{
// We only care that *something* has gone wrong, throw the first exception
throw aggEx.InnerException;
}
}
catch (Exception exc)
{
// also dispose all previously loaded correct PTables
for (int i = 0; i < tables.Count; ++i)
{
for (int j = 0; j < tables[i].Count; ++j)
{
if (tables[i][j] != null)
{
tables[i][j].Dispose();
}
}
}
throw new CorruptIndexException("Error while loading IndexMap.", exc);
}
return(tables);
}
public static IndexMap FromFile(string filename, Func <IndexEntry, bool> isHashCollision, int maxTablesPerLevel = 4, bool loadPTables = true)
{
var tables = new List <List <PTable> >();
int version;
long prepareCheckpoint = -1;
long commitCheckpoint = -1;
if (!File.Exists(filename))
{
return(new IndexMap(IndexMapVersion, tables, prepareCheckpoint, commitCheckpoint, isHashCollision, maxTablesPerLevel));
}
using (var f = File.OpenRead(filename))
using (var reader = new StreamReader(f))
{
// calculate real MD5 hash except first 32 bytes which are string representation of stored hash
f.Position = 32;
var realHash = MD5Hash.GetHashFor(f);
f.Position = 0;
// read stored MD5 hash and convert it from string to byte array
string text;
if ((text = reader.ReadLine()) == null)
{
throw new CorruptIndexException("IndexMap file is empty.");
}
if (text.Length != 32 || !text.All(x => char.IsDigit(x) || (x >= 'A' && x <= 'F')))
{
throw new CorruptIndexException("Corrupted MD5 hash.");
}
// check expected and real hashes are the same
var expectedHash = new byte[16];
for (int i = 0; i < 16; ++i)
{
expectedHash[i] = Convert.ToByte(text.Substring(i * 2, 2), 16);
}
if (expectedHash.Length != realHash.Length)
{
throw new InvalidOperationException("Invalid length of expected and real hash.");
}
for (int i = 0; i < realHash.Length; ++i)
{
if (expectedHash[i] != realHash[i])
{
throw new CorruptIndexException("Expected and real hash are different.");
}
}
// at this point we can assume the format is ok, so actually no need to check errors.
if ((text = reader.ReadLine()) == null)
{
throw new CorruptIndexException("Corrupted version.");
}
version = int.Parse(text);
// read and check prepare/commit checkpoint
if ((text = reader.ReadLine()) == null)
{
throw new CorruptIndexException("Corrupted commit checkpoint.");
}
try
{
var checkpoints = text.Split('/');
if (!long.TryParse(checkpoints[0], out prepareCheckpoint) || prepareCheckpoint < -1)
{
throw new CorruptIndexException("Invalid prepare checkpoint.");
}
if (!long.TryParse(checkpoints[1], out commitCheckpoint) || commitCheckpoint < -1)
{
throw new CorruptIndexException("Invalid commit checkpoint.");
}
}
catch (Exception exc)
{
throw new CorruptIndexException("Corrupted prepare/commit checkpoints pair.", exc);
}
// all next lines are PTables sorted by levels
while ((text = reader.ReadLine()) != null)
{
if (prepareCheckpoint < 0 || commitCheckpoint < 0)
{
throw new CorruptIndexException("Negative prepare/commit checkpoint in non-empty IndexMap.");
}
if (!loadPTables)
{
break;
}
PTable ptable = null;
var pieces = text.Split(',');
try
{
var level = int.Parse(pieces[0]);
var position = int.Parse(pieces[1]);
var file = pieces[2];
var path = Path.GetDirectoryName(filename);
var ptablePath = Path.Combine(path, file);
ptable = PTable.FromFile(ptablePath);
ptable.VerifyFileHash();
CreateIfNeeded(level, tables);
tables[level].Insert(position, ptable);
}
catch (Exception exc)
{
// if PTable file path was correct, but data is corrupted, we still need to dispose opened streams
if (ptable != null)
{
ptable.Dispose();
}
// also dispose all previously loaded correct PTables
for (int i = 0; i < tables.Count; ++i)
{
for (int j = 0; j < tables[i].Count; ++j)
{
tables[i][j].Dispose();
}
}
throw new CorruptIndexException("Error while loading IndexMap.", exc);
}
}
}
return(new IndexMap(version, tables, prepareCheckpoint, commitCheckpoint, isHashCollision, maxTablesPerLevel));
}