public static IndexMap FromFile( string filename, int maxTablesPerLevel, bool loadPTables, int cacheDepth, bool skipIndexVerify, int threads, int maxAutoMergeLevel, int pTableMaxReaderCount) { if (!File.Exists(filename)) { return(CreateEmpty(maxTablesPerLevel, maxAutoMergeLevel, pTableMaxReaderCount)); } using (var f = File.OpenRead(filename)) { // calculate real MD5 hash except first 32 bytes which are string representation of stored hash f.Position = 32; var realHash = MD5Hash.GetHashFor(f); f.Position = 0; using (var reader = new StreamReader(f)) { ReadAndCheckHash(reader, realHash); // at this point we can assume the format is ok, so actually no need to check errors. var version = ReadVersion(reader); var checkpoints = ReadCheckpoints(reader); var prepareCheckpoint = checkpoints.PreparePosition; var commitCheckpoint = checkpoints.CommitPosition; if (version > 1) { var tmpMaxAutoMergeLevel = ReadMaxAutoMergeLevel(reader); if (tmpMaxAutoMergeLevel < maxAutoMergeLevel) { throw new CorruptIndexException( $"Index map has lower maximum auto merge level ({tmpMaxAutoMergeLevel}) than is currently configured ({maxAutoMergeLevel}) and the index will need to be rebuilt"); } maxAutoMergeLevel = Math.Min(maxAutoMergeLevel, tmpMaxAutoMergeLevel); } //we are doing a logical upgrade of the version to have the new data, so we will change the version to match so that new files are saved with the right version version = IndexMapVersion; var tables = loadPTables ? LoadPTables(reader, filename, checkpoints, cacheDepth, skipIndexVerify, threads, pTableMaxReaderCount) : new List <List <PTable> >(); if (!loadPTables && reader.ReadLine() != null) { throw new CorruptIndexException( string.Format("Negative prepare/commit checkpoint in non-empty IndexMap: {0}.", checkpoints)); } return(new IndexMap(version, tables, prepareCheckpoint, commitCheckpoint, maxTablesPerLevel, maxAutoMergeLevel, pTableMaxReaderCount)); } } }
public void VerifyFileHash() { var stream = GetFileStream(); try { stream.Seek(0, SeekOrigin.Begin); var hash = MD5Hash.GetHashFor(stream, 0, stream.Length - MD5Size); stream.Seek(-MD5Size, SeekOrigin.End); var fileHash = new byte[MD5Size]; stream.Read(fileHash, 0, MD5Size); if (hash == null || fileHash.Length != hash.Length) { throw new CorruptIndexException(new HashValidationException()); } for (int i = 0; i < fileHash.Length; i++) { if (fileHash[i] != hash[i]) { throw new CorruptIndexException(new HashValidationException()); } } } finally { ReturnStream(stream); } }
private void CreateArtificialIndexMapFile(string filePath, long prepareCheckpoint, long commitCheckpoint, string ptablePath) { using (var memStream = new MemoryStream()) using (var memWriter = new StreamWriter(memStream)) { memWriter.WriteLine(new string('0', 32)); // pre-allocate space for MD5 hash memWriter.WriteLine("1"); memWriter.WriteLine("{0}/{1}", prepareCheckpoint, commitCheckpoint); if (!string.IsNullOrWhiteSpace(ptablePath)) { memWriter.WriteLine("0,0,{0}", ptablePath); } memWriter.Flush(); using (var f = File.OpenWrite(filePath)) using (var fileWriter = new StreamWriter(f)) { memStream.Position = 0; memStream.CopyTo(f); memStream.Position = 32; var hash = MD5Hash.GetHashFor(memStream); f.Position = 0; for (int i = 0; i < hash.Length; ++i) { fileWriter.Write(hash[i].ToString("X2")); } fileWriter.WriteLine(); fileWriter.Flush(); f.FlushToDisk(); } } }
private string GetDbPathHash(string dbPath) { using (var memStream = new MemoryStream(Helper.UTF8NoBom.GetBytes(dbPath))) { return(BitConverter.ToString(MD5Hash.GetHashFor(memStream)).Replace("-", "")); } }
public void SaveToFile(string filename) { var tmpIndexMap = string.Format("{0}.{1}.indexmap.tmp", filename, Guid.NewGuid()); using (var memStream = new MemoryStream()) using (var memWriter = new StreamWriter(memStream)) { memWriter.WriteLine(new string('0', 32)); // pre-allocate space for MD5 hash memWriter.WriteLine(Version); memWriter.WriteLine("{0}/{1}", PrepareCheckpoint, CommitCheckpoint); for (int i = 0; i < _map.Count; i++) { for (int j = 0; j < _map[i].Count; j++) { memWriter.WriteLine("{0},{1},{2}", i, j, new FileInfo(_map[i][j].Filename).Name); } } memWriter.Flush(); memStream.Position = 32; var hash = MD5Hash.GetHashFor(memStream); memStream.Position = 0; foreach (var t in hash) { memWriter.Write(t.ToString("X2")); } memWriter.Flush(); memStream.Position = 0; using (var f = File.OpenWrite(tmpIndexMap)) { f.Write(memStream.GetBuffer(), 0, (int)memStream.Length); f.FlushToDisk(); } } int trial = 0; while (trial < 5) { try { if (File.Exists(filename)) { File.Delete(filename); } File.Move(tmpIndexMap, filename); break; } catch (IOException exc) { Log.DebugException(exc, "Failed trial to replace indexmap."); trial += 1; } } }
public void VerifyFileHash() { var sw = Stopwatch.StartNew(); Log.Trace("Verifying file hash of PTable '{0}' started...", Path.GetFileName(Filename)); #if __MonoCS__ var workItem = GetWorkItem(); var stream = workItem.Stream; try { #else using (var stream = UnbufferedFileReadStream.Open(_filename)) { #endif //stream.Position = 0; var hash = MD5Hash.GetHashFor(stream, 0, stream.Length - MD5Size); var fileHash = new byte[MD5Size]; stream.Read(fileHash, 0, MD5Size); if (hash == null) { throw new CorruptIndexException(new HashValidationException("Calculated MD5 hash is null!")); } if (fileHash.Length != hash.Length) { throw new CorruptIndexException( new HashValidationException( string.Format( "Hash sizes differ! FileHash({0}): {1}, hash({2}): {3}.", fileHash.Length, BitConverter.ToString(fileHash), hash.Length, BitConverter.ToString(hash)))); } for (int i = 0; i < fileHash.Length; i++) { if (fileHash[i] != hash[i]) { throw new CorruptIndexException( new HashValidationException( string.Format( "Hashes are different! FileHash: {0}, hash: {1}.", BitConverter.ToString(fileHash), BitConverter.ToString(hash)))); } } } #if __MonoCS__ finally { ReturnWorkItem(workItem); } #endif Log.Trace("Verifying file hash of PTable '{0}' ({1} entries) done in {2}.", Path.GetFileName(Filename), Count, sw.Elapsed); }
public override void TestFixtureSetUp() { var ms = new MemoryStream(Encoding.UTF8.GetBytes(V2FileContents)); var md5 = MD5Hash.GetHashFor(ms); V2FileContents = BitConverter.ToString(md5).Replace("-", "") + V2FileContents; base.TestFixtureSetUp(); _filename = GetFilePathFor("indexfile"); File.WriteAllText(_filename, V1FileContents); }
public void includes_correct_substream_data() { var bytes = new byte[1024]; for (int i = 15; i < 1024; i++) { bytes[i] = (byte)(i % 255); } var stream = new MemoryStream(bytes); var hash = MD5Hash.GetHashFor(stream, 16, bytes.Length - 32); using (var md5 = MD5.Create()) { var referenceHash = md5.ComputeHash(bytes, 16, bytes.Length - 32); Assert.AreEqual(16, hash.Length); Assert.AreEqual(referenceHash, hash); } }
public void changing_data_in_stream_results_in_different_hash() { var bytes = new byte[1024]; for (int i = 15; i < 1024; i++) { bytes[i] = (byte)(i % 255); } var stream = new MemoryStream(bytes); stream.Seek(16, SeekOrigin.Begin); var hash = MD5Hash.GetHashFor(stream); bytes[243] = 17; stream.Seek(16, SeekOrigin.Begin); var hash2 = MD5Hash.GetHashFor(stream); Assert.AreNotEqual(hash, hash2); }
public void SaveToFile(string filename) { var tmpIndexMap = string.Format("{0}.{1}.indexmap.tmp", filename, Guid.NewGuid()); using (var memStream = new MemoryStream()) using (var memWriter = new StreamWriter(memStream)) { memWriter.WriteLine(new string('0', 32)); // pre-allocate space for MD5 hash memWriter.WriteLine(Version); memWriter.WriteLine("{0}/{1}", PrepareCheckpoint, CommitCheckpoint); for (int i = 0; i < _map.Count; i++) { for (int j = 0; j < _map[i].Count; j++) { memWriter.WriteLine("{0},{1},{2}", i, j, new FileInfo(_map[i][j].Filename).Name); } } memWriter.Flush(); using (var f = File.OpenWrite(tmpIndexMap)) using (var fileWriter = new StreamWriter(f)) { memStream.Position = 0; memStream.CopyTo(f); memStream.Position = 32; var hash = MD5Hash.GetHashFor(memStream); f.Position = 0; for (int i = 0; i < hash.Length; ++i) { fileWriter.Write(hash[i].ToString("X2")); } fileWriter.WriteLine(); fileWriter.Flush(); } } if (File.Exists(filename)) { File.Delete(filename); } File.Move(tmpIndexMap, filename); }
public void the_file_contains_correct_data() { using (var fs = File.OpenRead(_filename)) using (var reader = new StreamReader(fs)) { var text = reader.ReadToEnd(); var lines = text.Replace("\r", "").Split('\n'); fs.Position = 32; var md5 = MD5Hash.GetHashFor(fs); var md5String = BitConverter.ToString(md5).Replace("-", ""); Assert.AreEqual(5, lines.Count()); Assert.AreEqual(md5String, lines[0]); Assert.AreEqual(IndexMap.IndexMapVersion.ToString(), lines[1]); Assert.AreEqual("-1/-1", lines[2]); Assert.AreEqual($"{int.MaxValue}", lines[3]); Assert.AreEqual("", lines[4]); } }
public void the_file_contains_correct_data() { using (var fs = File.OpenRead(_filename)) using (var reader = new StreamReader(fs)) { var text = reader.ReadToEnd(); var lines = text.Replace("\r", "").Split('\n'); fs.Position = 32; var md5 = MD5Hash.GetHashFor(fs); var md5String = BitConverter.ToString(md5).Replace("-", ""); Assert.AreEqual(6, lines.Count()); Assert.AreEqual(md5String, lines[0]); Assert.AreEqual(_map.Version.ToString(), lines[1]); Assert.AreEqual("7/11", lines[2]); Assert.AreEqual(_maxAutoMergeIndexLevel.ToString(), lines[3]); Assert.AreEqual("0,0," + Path.GetFileName(_tablename), lines[4]); Assert.AreEqual("", lines[5]); } }
public void does_not_include_previous_data_in_stream() { var bytes = new byte[1024]; for (int i = 15; i < 1024; i++) { bytes[i] = (byte)(i % 255); } var stream = new MemoryStream(bytes); stream.Seek(16, SeekOrigin.Begin); var hash = MD5Hash.GetHashFor(stream); Array.Copy(hash, 0, bytes, 0, hash.Length); stream.Seek(16, SeekOrigin.Begin); var hash2 = MD5Hash.GetHashFor(stream); Assert.AreEqual(16, hash.Length); Assert.AreEqual(hash, hash2); }
public void randomized_hash_verification_test() { var buf = new byte[1024]; var seed = Environment.TickCount; Console.WriteLine("Seed: {0}", seed); new Random().NextBytes(buf); var stream = new MemoryStream(buf); using (var md5 = MD5.Create()) { for (int i = 0; i < buf.Length; ++i) { for (int j = i; j < buf.Length + 10; ++j) { var referenceHash = md5.ComputeHash(buf, i, Math.Min(buf.Length - i, j - i + 1)); var hash = MD5Hash.GetHashFor(stream, i, j - i + 1); Assert.AreEqual(16, hash.Length); Assert.AreEqual(referenceHash, hash); } } } }
public static IndexMap FromFile(string filename, int maxTablesPerLevel = 4, bool loadPTables = true, int cacheDepth = 16, bool skipIndexVerify = false, int threads = 1) { if (!File.Exists(filename)) { return(CreateEmpty(maxTablesPerLevel)); } using (var f = File.OpenRead(filename)) { // calculate real MD5 hash except first 32 bytes which are string representation of stored hash f.Position = 32; var realHash = MD5Hash.GetHashFor(f); f.Position = 0; using (var reader = new StreamReader(f)) { ReadAndCheckHash(reader, realHash); // at this point we can assume the format is ok, so actually no need to check errors. var version = ReadVersion(reader); var checkpoints = ReadCheckpoints(reader); var prepareCheckpoint = checkpoints.PreparePosition; var commitCheckpoint = checkpoints.CommitPosition; var tables = loadPTables ? LoadPTables(reader, filename, checkpoints, cacheDepth, skipIndexVerify, threads) : new List <List <PTable> >(); if (!loadPTables && reader.ReadLine() != null) { throw new CorruptIndexException( string.Format("Negative prepare/commit checkpoint in non-empty IndexMap: {0}.", checkpoints)); } return(new IndexMap(version, tables, prepareCheckpoint, commitCheckpoint, maxTablesPerLevel)); } } }
public void SaveToFile(string filename) { var tmpIndexMap = string.Format("{0}.{1}.indexmap.tmp", filename, Guid.NewGuid()); using (var memStream = new MemoryStream()) using (var memWriter = new StreamWriter(memStream)) { memWriter.WriteLine(new string('0', 32)); // pre-allocate space for MD5 hash memWriter.WriteLine(Version); memWriter.WriteLine("{0}/{1}", PrepareCheckpoint, CommitCheckpoint); for (int i = 0; i < _map.Count; i++) { for (int j = 0; j < _map[i].Count; j++) { memWriter.WriteLine("{0},{1},{2}", i, j, new FileInfo(_map[i][j].Filename).Name); } } memWriter.Flush(); memStream.Position = 32; var hash = MD5Hash.GetHashFor(memStream); memStream.Position = 0; foreach (var t in hash) { memWriter.Write(t.ToString("X2")); } memWriter.Flush(); memStream.Position = 0; using (var f = File.OpenWrite(tmpIndexMap)) { f.Write(memStream.GetBuffer(), 0, (int)memStream.Length); f.FlushToDisk(); } } int trial = 0; int maxTrials = 5; while (trial < maxTrials) { Action <Exception> errorHandler = ex => { Log.Error("Failed trial to replace indexmap {indexMap} with {tmpIndexMap}.", filename, tmpIndexMap); Log.Error("Exception: {e}", ex); trial += 1; }; try { if (File.Exists(filename)) { File.SetAttributes(filename, FileAttributes.Normal); File.Delete(filename); } File.Move(tmpIndexMap, filename); break; } catch (IOException exc) { errorHandler(exc); if (trial >= maxTrials) { ProcessUtil.PrintWhoIsLocking(tmpIndexMap, Log); ProcessUtil.PrintWhoIsLocking(filename, Log); } } catch (UnauthorizedAccessException exc) { errorHandler(exc); } } }
public static IndexMap FromFile(string filename, Func <IndexEntry, bool> isHashCollision, int maxTablesPerLevel = 4, bool loadPTables = true) { var tables = new List <List <PTable> >(); int version; long prepareCheckpoint = -1; long commitCheckpoint = -1; if (!File.Exists(filename)) { return(new IndexMap(IndexMapVersion, tables, prepareCheckpoint, commitCheckpoint, isHashCollision, maxTablesPerLevel)); } using (var f = File.OpenRead(filename)) using (var reader = new StreamReader(f)) { // calculate real MD5 hash except first 32 bytes which are string representation of stored hash f.Position = 32; var realHash = MD5Hash.GetHashFor(f); f.Position = 0; // read stored MD5 hash and convert it from string to byte array string text; if ((text = reader.ReadLine()) == null) { throw new CorruptIndexException("IndexMap file is empty."); } if (text.Length != 32 || !text.All(x => char.IsDigit(x) || (x >= 'A' && x <= 'F'))) { throw new CorruptIndexException("Corrupted MD5 hash."); } // check expected and real hashes are the same var expectedHash = new byte[16]; for (int i = 0; i < 16; ++i) { expectedHash[i] = Convert.ToByte(text.Substring(i * 2, 2), 16); } if (expectedHash.Length != realHash.Length) { throw new InvalidOperationException("Invalid length of expected and real hash."); } for (int i = 0; i < realHash.Length; ++i) { if (expectedHash[i] != realHash[i]) { throw new CorruptIndexException("Expected and real hash are different."); } } // at this point we can assume the format is ok, so actually no need to check errors. if ((text = reader.ReadLine()) == null) { throw new CorruptIndexException("Corrupted version."); } version = int.Parse(text); // read and check prepare/commit checkpoint if ((text = reader.ReadLine()) == null) { throw new CorruptIndexException("Corrupted commit checkpoint."); } try { var checkpoints = text.Split('/'); if (!long.TryParse(checkpoints[0], out prepareCheckpoint) || prepareCheckpoint < -1) { throw new CorruptIndexException("Invalid prepare checkpoint."); } if (!long.TryParse(checkpoints[1], out commitCheckpoint) || commitCheckpoint < -1) { throw new CorruptIndexException("Invalid commit checkpoint."); } } catch (Exception exc) { throw new CorruptIndexException("Corrupted prepare/commit checkpoints pair.", exc); } // all next lines are PTables sorted by levels while ((text = reader.ReadLine()) != null) { if (prepareCheckpoint < 0 || commitCheckpoint < 0) { throw new CorruptIndexException("Negative prepare/commit checkpoint in non-empty IndexMap."); } if (!loadPTables) { break; } PTable ptable = null; var pieces = text.Split(','); try { var level = int.Parse(pieces[0]); var position = int.Parse(pieces[1]); var file = pieces[2]; var path = Path.GetDirectoryName(filename); var ptablePath = Path.Combine(path, file); ptable = PTable.FromFile(ptablePath); ptable.VerifyFileHash(); CreateIfNeeded(level, tables); tables[level].Insert(position, ptable); } catch (Exception exc) { // if PTable file path was correct, but data is corrupted, we still need to dispose opened streams if (ptable != null) { ptable.Dispose(); } // also dispose all previously loaded correct PTables for (int i = 0; i < tables.Count; ++i) { for (int j = 0; j < tables[i].Count; ++j) { tables[i][j].Dispose(); } } throw new CorruptIndexException("Error while loading IndexMap.", exc); } } } return(new IndexMap(version, tables, prepareCheckpoint, commitCheckpoint, isHashCollision, maxTablesPerLevel)); }