public static IndexMap FromFile(
string filename,
int maxTablesPerLevel,
bool loadPTables,
int cacheDepth,
bool skipIndexVerify,
int threads,
int maxAutoMergeLevel,
int pTableMaxReaderCount)
{
if (!File.Exists(filename))
{
return(CreateEmpty(maxTablesPerLevel, maxAutoMergeLevel, pTableMaxReaderCount));
}
using (var f = File.OpenRead(filename)) {
// calculate real MD5 hash except first 32 bytes which are string representation of stored hash
f.Position = 32;
var realHash = MD5Hash.GetHashFor(f);
f.Position = 0;
using (var reader = new StreamReader(f)) {
ReadAndCheckHash(reader, realHash);
// at this point we can assume the format is ok, so actually no need to check errors.
var version = ReadVersion(reader);
var checkpoints = ReadCheckpoints(reader);
var prepareCheckpoint = checkpoints.PreparePosition;
var commitCheckpoint = checkpoints.CommitPosition;
if (version > 1)
{
var tmpMaxAutoMergeLevel = ReadMaxAutoMergeLevel(reader);
if (tmpMaxAutoMergeLevel < maxAutoMergeLevel)
{
throw new CorruptIndexException(
$"Index map has lower maximum auto merge level ({tmpMaxAutoMergeLevel}) than is currently configured ({maxAutoMergeLevel}) and the index will need to be rebuilt");
}
maxAutoMergeLevel = Math.Min(maxAutoMergeLevel, tmpMaxAutoMergeLevel);
}
//we are doing a logical upgrade of the version to have the new data, so we will change the version to match so that new files are saved with the right version
version = IndexMapVersion;
var tables = loadPTables
? LoadPTables(reader, filename, checkpoints, cacheDepth, skipIndexVerify, threads, pTableMaxReaderCount)
: new List <List <PTable> >();
if (!loadPTables && reader.ReadLine() != null)
{
throw new CorruptIndexException(
string.Format("Negative prepare/commit checkpoint in non-empty IndexMap: {0}.",
checkpoints));
}
return(new IndexMap(version, tables, prepareCheckpoint, commitCheckpoint, maxTablesPerLevel,
maxAutoMergeLevel, pTableMaxReaderCount));
}
}
}
public void VerifyFileHash()
{
var stream = GetFileStream();
try
{
stream.Seek(0, SeekOrigin.Begin);
var hash = MD5Hash.GetHashFor(stream, 0, stream.Length - MD5Size);
stream.Seek(-MD5Size, SeekOrigin.End);
var fileHash = new byte[MD5Size];
stream.Read(fileHash, 0, MD5Size);
if (hash == null || fileHash.Length != hash.Length)
{
throw new CorruptIndexException(new HashValidationException());
}
for (int i = 0; i < fileHash.Length; i++)
{
if (fileHash[i] != hash[i])
{
throw new CorruptIndexException(new HashValidationException());
}
}
}
finally
{
ReturnStream(stream);
}
}
private void CreateArtificialIndexMapFile(string filePath, long prepareCheckpoint, long commitCheckpoint, string ptablePath)
{
using (var memStream = new MemoryStream())
using (var memWriter = new StreamWriter(memStream))
{
memWriter.WriteLine(new string('0', 32)); // pre-allocate space for MD5 hash
memWriter.WriteLine("1");
memWriter.WriteLine("{0}/{1}", prepareCheckpoint, commitCheckpoint);
if (!string.IsNullOrWhiteSpace(ptablePath))
{
memWriter.WriteLine("0,0,{0}", ptablePath);
}
memWriter.Flush();
using (var f = File.OpenWrite(filePath))
using (var fileWriter = new StreamWriter(f))
{
memStream.Position = 0;
memStream.CopyTo(f);
memStream.Position = 32;
var hash = MD5Hash.GetHashFor(memStream);
f.Position = 0;
for (int i = 0; i < hash.Length; ++i)
{
fileWriter.Write(hash[i].ToString("X2"));
}
fileWriter.WriteLine();
fileWriter.Flush();
f.FlushToDisk();
}
}
}
private string GetDbPathHash(string dbPath)
{
using (var memStream = new MemoryStream(Helper.UTF8NoBom.GetBytes(dbPath)))
{
return(BitConverter.ToString(MD5Hash.GetHashFor(memStream)).Replace("-", ""));
}
}
public void SaveToFile(string filename)
{
var tmpIndexMap = string.Format("{0}.{1}.indexmap.tmp", filename, Guid.NewGuid());
using (var memStream = new MemoryStream())
using (var memWriter = new StreamWriter(memStream))
{
memWriter.WriteLine(new string('0', 32)); // pre-allocate space for MD5 hash
memWriter.WriteLine(Version);
memWriter.WriteLine("{0}/{1}", PrepareCheckpoint, CommitCheckpoint);
for (int i = 0; i < _map.Count; i++)
{
for (int j = 0; j < _map[i].Count; j++)
{
memWriter.WriteLine("{0},{1},{2}", i, j, new FileInfo(_map[i][j].Filename).Name);
}
}
memWriter.Flush();
memStream.Position = 32;
var hash = MD5Hash.GetHashFor(memStream);
memStream.Position = 0;
foreach (var t in hash)
{
memWriter.Write(t.ToString("X2"));
}
memWriter.Flush();
memStream.Position = 0;
using (var f = File.OpenWrite(tmpIndexMap))
{
f.Write(memStream.GetBuffer(), 0, (int)memStream.Length);
f.FlushToDisk();
}
}
int trial = 0;
while (trial < 5)
{
try
{
if (File.Exists(filename))
{
File.Delete(filename);
}
File.Move(tmpIndexMap, filename);
break;
}
catch (IOException exc)
{
Log.DebugException(exc, "Failed trial to replace indexmap.");
trial += 1;
}
}
}
public void VerifyFileHash()
{
var sw = Stopwatch.StartNew();
Log.Trace("Verifying file hash of PTable '{0}' started...", Path.GetFileName(Filename));
#if __MonoCS__
var workItem = GetWorkItem();
var stream = workItem.Stream;
try {
#else
using (var stream = UnbufferedFileReadStream.Open(_filename))
{
#endif
//stream.Position = 0;
var hash = MD5Hash.GetHashFor(stream, 0, stream.Length - MD5Size);
var fileHash = new byte[MD5Size];
stream.Read(fileHash, 0, MD5Size);
if (hash == null)
{
throw new CorruptIndexException(new HashValidationException("Calculated MD5 hash is null!"));
}
if (fileHash.Length != hash.Length)
{
throw new CorruptIndexException(
new HashValidationException(
string.Format(
"Hash sizes differ! FileHash({0}): {1}, hash({2}): {3}.",
fileHash.Length,
BitConverter.ToString(fileHash),
hash.Length,
BitConverter.ToString(hash))));
}
for (int i = 0; i < fileHash.Length; i++)
{
if (fileHash[i] != hash[i])
{
throw new CorruptIndexException(
new HashValidationException(
string.Format(
"Hashes are different! FileHash: {0}, hash: {1}.",
BitConverter.ToString(fileHash),
BitConverter.ToString(hash))));
}
}
}
#if __MonoCS__
finally
{
ReturnWorkItem(workItem);
}
#endif
Log.Trace("Verifying file hash of PTable '{0}' ({1} entries) done in {2}.", Path.GetFileName(Filename), Count, sw.Elapsed);
}
public override void TestFixtureSetUp()
{
var ms = new MemoryStream(Encoding.UTF8.GetBytes(V2FileContents));
var md5 = MD5Hash.GetHashFor(ms);
V2FileContents = BitConverter.ToString(md5).Replace("-", "") + V2FileContents;
base.TestFixtureSetUp();
_filename = GetFilePathFor("indexfile");
File.WriteAllText(_filename, V1FileContents);
}
public void includes_correct_substream_data()
{
var bytes = new byte[1024];
for (int i = 15; i < 1024; i++)
{
bytes[i] = (byte)(i % 255);
}
var stream = new MemoryStream(bytes);
var hash = MD5Hash.GetHashFor(stream, 16, bytes.Length - 32);
using (var md5 = MD5.Create())
{
var referenceHash = md5.ComputeHash(bytes, 16, bytes.Length - 32);
Assert.AreEqual(16, hash.Length);
Assert.AreEqual(referenceHash, hash);
}
}
public void changing_data_in_stream_results_in_different_hash()
{
var bytes = new byte[1024];
for (int i = 15; i < 1024; i++)
{
bytes[i] = (byte)(i % 255);
}
var stream = new MemoryStream(bytes);
stream.Seek(16, SeekOrigin.Begin);
var hash = MD5Hash.GetHashFor(stream);
bytes[243] = 17;
stream.Seek(16, SeekOrigin.Begin);
var hash2 = MD5Hash.GetHashFor(stream);
Assert.AreNotEqual(hash, hash2);
}
public void SaveToFile(string filename)
{
var tmpIndexMap = string.Format("{0}.{1}.indexmap.tmp", filename, Guid.NewGuid());
using (var memStream = new MemoryStream())
using (var memWriter = new StreamWriter(memStream))
{
memWriter.WriteLine(new string('0', 32)); // pre-allocate space for MD5 hash
memWriter.WriteLine(Version);
memWriter.WriteLine("{0}/{1}", PrepareCheckpoint, CommitCheckpoint);
for (int i = 0; i < _map.Count; i++)
{
for (int j = 0; j < _map[i].Count; j++)
{
memWriter.WriteLine("{0},{1},{2}", i, j, new FileInfo(_map[i][j].Filename).Name);
}
}
memWriter.Flush();
using (var f = File.OpenWrite(tmpIndexMap))
using (var fileWriter = new StreamWriter(f))
{
memStream.Position = 0;
memStream.CopyTo(f);
memStream.Position = 32;
var hash = MD5Hash.GetHashFor(memStream);
f.Position = 0;
for (int i = 0; i < hash.Length; ++i)
{
fileWriter.Write(hash[i].ToString("X2"));
}
fileWriter.WriteLine();
fileWriter.Flush();
}
}
if (File.Exists(filename))
{
File.Delete(filename);
}
File.Move(tmpIndexMap, filename);
}
public void the_file_contains_correct_data()
{
using (var fs = File.OpenRead(_filename))
using (var reader = new StreamReader(fs)) {
var text = reader.ReadToEnd();
var lines = text.Replace("\r", "").Split('\n');
fs.Position = 32;
var md5 = MD5Hash.GetHashFor(fs);
var md5String = BitConverter.ToString(md5).Replace("-", "");
Assert.AreEqual(5, lines.Count());
Assert.AreEqual(md5String, lines[0]);
Assert.AreEqual(IndexMap.IndexMapVersion.ToString(), lines[1]);
Assert.AreEqual("-1/-1", lines[2]);
Assert.AreEqual($"{int.MaxValue}", lines[3]);
Assert.AreEqual("", lines[4]);
}
}
public void the_file_contains_correct_data()
{
using (var fs = File.OpenRead(_filename))
using (var reader = new StreamReader(fs)) {
var text = reader.ReadToEnd();
var lines = text.Replace("\r", "").Split('\n');
fs.Position = 32;
var md5 = MD5Hash.GetHashFor(fs);
var md5String = BitConverter.ToString(md5).Replace("-", "");
Assert.AreEqual(6, lines.Count());
Assert.AreEqual(md5String, lines[0]);
Assert.AreEqual(_map.Version.ToString(), lines[1]);
Assert.AreEqual("7/11", lines[2]);
Assert.AreEqual(_maxAutoMergeIndexLevel.ToString(), lines[3]);
Assert.AreEqual("0,0," + Path.GetFileName(_tablename), lines[4]);
Assert.AreEqual("", lines[5]);
}
}
public void does_not_include_previous_data_in_stream()
{
var bytes = new byte[1024];
for (int i = 15; i < 1024; i++)
{
bytes[i] = (byte)(i % 255);
}
var stream = new MemoryStream(bytes);
stream.Seek(16, SeekOrigin.Begin);
var hash = MD5Hash.GetHashFor(stream);
Array.Copy(hash, 0, bytes, 0, hash.Length);
stream.Seek(16, SeekOrigin.Begin);
var hash2 = MD5Hash.GetHashFor(stream);
Assert.AreEqual(16, hash.Length);
Assert.AreEqual(hash, hash2);
}
public void randomized_hash_verification_test()
{
var buf = new byte[1024];
var seed = Environment.TickCount;
Console.WriteLine("Seed: {0}", seed);
new Random().NextBytes(buf);
var stream = new MemoryStream(buf);
using (var md5 = MD5.Create()) {
for (int i = 0; i < buf.Length; ++i)
{
for (int j = i; j < buf.Length + 10; ++j)
{
var referenceHash = md5.ComputeHash(buf, i, Math.Min(buf.Length - i, j - i + 1));
var hash = MD5Hash.GetHashFor(stream, i, j - i + 1);
Assert.AreEqual(16, hash.Length);
Assert.AreEqual(referenceHash, hash);
}
}
}
}
public static IndexMap FromFile(string filename, int maxTablesPerLevel = 4, bool loadPTables = true, int cacheDepth = 16, bool skipIndexVerify = false,
int threads = 1)
{
if (!File.Exists(filename))
{
return(CreateEmpty(maxTablesPerLevel));
}
using (var f = File.OpenRead(filename))
{
// calculate real MD5 hash except first 32 bytes which are string representation of stored hash
f.Position = 32;
var realHash = MD5Hash.GetHashFor(f);
f.Position = 0;
using (var reader = new StreamReader(f))
{
ReadAndCheckHash(reader, realHash);
// at this point we can assume the format is ok, so actually no need to check errors.
var version = ReadVersion(reader);
var checkpoints = ReadCheckpoints(reader);
var prepareCheckpoint = checkpoints.PreparePosition;
var commitCheckpoint = checkpoints.CommitPosition;
var tables = loadPTables ? LoadPTables(reader, filename, checkpoints, cacheDepth, skipIndexVerify, threads) : new List <List <PTable> >();
if (!loadPTables && reader.ReadLine() != null)
{
throw new CorruptIndexException(
string.Format("Negative prepare/commit checkpoint in non-empty IndexMap: {0}.", checkpoints));
}
return(new IndexMap(version, tables, prepareCheckpoint, commitCheckpoint, maxTablesPerLevel));
}
}
}
public void SaveToFile(string filename)
{
var tmpIndexMap = string.Format("{0}.{1}.indexmap.tmp", filename, Guid.NewGuid());
using (var memStream = new MemoryStream())
using (var memWriter = new StreamWriter(memStream))
{
memWriter.WriteLine(new string('0', 32)); // pre-allocate space for MD5 hash
memWriter.WriteLine(Version);
memWriter.WriteLine("{0}/{1}", PrepareCheckpoint, CommitCheckpoint);
for (int i = 0; i < _map.Count; i++)
{
for (int j = 0; j < _map[i].Count; j++)
{
memWriter.WriteLine("{0},{1},{2}", i, j, new FileInfo(_map[i][j].Filename).Name);
}
}
memWriter.Flush();
memStream.Position = 32;
var hash = MD5Hash.GetHashFor(memStream);
memStream.Position = 0;
foreach (var t in hash)
{
memWriter.Write(t.ToString("X2"));
}
memWriter.Flush();
memStream.Position = 0;
using (var f = File.OpenWrite(tmpIndexMap))
{
f.Write(memStream.GetBuffer(), 0, (int)memStream.Length);
f.FlushToDisk();
}
}
int trial = 0;
int maxTrials = 5;
while (trial < maxTrials)
{
Action <Exception> errorHandler = ex =>
{
Log.Error("Failed trial to replace indexmap {indexMap} with {tmpIndexMap}.", filename, tmpIndexMap);
Log.Error("Exception: {e}", ex);
trial += 1;
};
try
{
if (File.Exists(filename))
{
File.SetAttributes(filename, FileAttributes.Normal);
File.Delete(filename);
}
File.Move(tmpIndexMap, filename);
break;
}
catch (IOException exc)
{
errorHandler(exc);
if (trial >= maxTrials)
{
ProcessUtil.PrintWhoIsLocking(tmpIndexMap, Log);
ProcessUtil.PrintWhoIsLocking(filename, Log);
}
}
catch (UnauthorizedAccessException exc)
{
errorHandler(exc);
}
}
}
public static IndexMap FromFile(string filename, Func <IndexEntry, bool> isHashCollision, int maxTablesPerLevel = 4, bool loadPTables = true)
{
var tables = new List <List <PTable> >();
int version;
long prepareCheckpoint = -1;
long commitCheckpoint = -1;
if (!File.Exists(filename))
{
return(new IndexMap(IndexMapVersion, tables, prepareCheckpoint, commitCheckpoint, isHashCollision, maxTablesPerLevel));
}
using (var f = File.OpenRead(filename))
using (var reader = new StreamReader(f))
{
// calculate real MD5 hash except first 32 bytes which are string representation of stored hash
f.Position = 32;
var realHash = MD5Hash.GetHashFor(f);
f.Position = 0;
// read stored MD5 hash and convert it from string to byte array
string text;
if ((text = reader.ReadLine()) == null)
{
throw new CorruptIndexException("IndexMap file is empty.");
}
if (text.Length != 32 || !text.All(x => char.IsDigit(x) || (x >= 'A' && x <= 'F')))
{
throw new CorruptIndexException("Corrupted MD5 hash.");
}
// check expected and real hashes are the same
var expectedHash = new byte[16];
for (int i = 0; i < 16; ++i)
{
expectedHash[i] = Convert.ToByte(text.Substring(i * 2, 2), 16);
}
if (expectedHash.Length != realHash.Length)
{
throw new InvalidOperationException("Invalid length of expected and real hash.");
}
for (int i = 0; i < realHash.Length; ++i)
{
if (expectedHash[i] != realHash[i])
{
throw new CorruptIndexException("Expected and real hash are different.");
}
}
// at this point we can assume the format is ok, so actually no need to check errors.
if ((text = reader.ReadLine()) == null)
{
throw new CorruptIndexException("Corrupted version.");
}
version = int.Parse(text);
// read and check prepare/commit checkpoint
if ((text = reader.ReadLine()) == null)
{
throw new CorruptIndexException("Corrupted commit checkpoint.");
}
try
{
var checkpoints = text.Split('/');
if (!long.TryParse(checkpoints[0], out prepareCheckpoint) || prepareCheckpoint < -1)
{
throw new CorruptIndexException("Invalid prepare checkpoint.");
}
if (!long.TryParse(checkpoints[1], out commitCheckpoint) || commitCheckpoint < -1)
{
throw new CorruptIndexException("Invalid commit checkpoint.");
}
}
catch (Exception exc)
{
throw new CorruptIndexException("Corrupted prepare/commit checkpoints pair.", exc);
}
// all next lines are PTables sorted by levels
while ((text = reader.ReadLine()) != null)
{
if (prepareCheckpoint < 0 || commitCheckpoint < 0)
{
throw new CorruptIndexException("Negative prepare/commit checkpoint in non-empty IndexMap.");
}
if (!loadPTables)
{
break;
}
PTable ptable = null;
var pieces = text.Split(',');
try
{
var level = int.Parse(pieces[0]);
var position = int.Parse(pieces[1]);
var file = pieces[2];
var path = Path.GetDirectoryName(filename);
var ptablePath = Path.Combine(path, file);
ptable = PTable.FromFile(ptablePath);
ptable.VerifyFileHash();
CreateIfNeeded(level, tables);
tables[level].Insert(position, ptable);
}
catch (Exception exc)
{
// if PTable file path was correct, but data is corrupted, we still need to dispose opened streams
if (ptable != null)
{
ptable.Dispose();
}
// also dispose all previously loaded correct PTables
for (int i = 0; i < tables.Count; ++i)
{
for (int j = 0; j < tables[i].Count; ++j)
{
tables[i][j].Dispose();
}
}
throw new CorruptIndexException("Error while loading IndexMap.", exc);
}
}
}
return(new IndexMap(version, tables, prepareCheckpoint, commitCheckpoint, isHashCollision, maxTablesPerLevel));
}
