/// <summary> /// Get a page from cache or from disk (and put on cache) /// </summary> public T GetPage <T>(uint pageID, bool setDirty = false) where T : BasePage { var page = _cache.GetPage(pageID); // is not on cache? load from disk if (page == null) { var buffer = _disk.ReadPage(pageID); page = BasePage.ReadPage(buffer); _cache.AddPage(page); } #if DEBUG // if page is empty, convert to T if (page.PageType == PageType.Empty && typeof(T) != typeof(BasePage)) { throw new Exception("Pager.GetPage<T>() never shuld happend"); } #endif // set page as dirty if passing by param if (setDirty) { this.SetDirty((T)page); } return((T)page); }
/// <summary> /// Try recovery journal file (if exists). Restore original datafile /// Journal file are NOT encrypted (even when datafile are encrypted) /// </summary> public void Recovery() { var fileSize = _disk.FileLength; // read all journal pages foreach (var buffer in _disk.ReadJournal()) { // read pageID (first 4 bytes) var pageID = BitConverter.ToUInt32(buffer, 0); _log.Write(Logger.RECOVERY, "recover page #{0:0000}", pageID); // if header, read all byte (to get original filesize) if (pageID == 0) { var header = (HeaderPage)BasePage.ReadPage(buffer); fileSize = BasePage.GetSizeOfPages(header.LastPageID + 1); } // write in stream (encrypt if datafile is encrypted) _disk.WritePage(pageID, _crypto == null || pageID == 0 ? buffer : _crypto.Encrypt(buffer)); } _log.Write(Logger.RECOVERY, "resize datafile to {0} bytes", fileSize); // redim filesize if grow more than original before rollback _disk.SetLength(fileSize); // empty journal file _disk.ClearJournal(); }
/// <summary> /// Read page bytes from disk /// </summary> public virtual byte[] ReadPage(uint pageID) { var buffer = new byte[BasePage.PAGE_SIZE]; var position = BasePage.GetSizeOfPages(pageID); // position cursor if (_stream.Position != position) { _stream.Seek(position, SeekOrigin.Begin); } // read bytes from data file _stream.Read(buffer, 0, BasePage.PAGE_SIZE); _log.Write(Logger.DISK, "read page #{0:0000} :: {1}", pageID, (PageType)buffer[PAGE_TYPE_POSITION]); // when read header, checks passoword if (pageID == 0) { // I know, header page will be double read (it's the price for isolated concerns) var header = (HeaderPage)BasePage.ReadPage(buffer); ValidatePassword(header.Password); } return(buffer); }
private void Recovery(FileStream journal) { var fileSize = _stream.Length; var buffer = new byte[BasePage.PAGE_SIZE]; journal.Seek(0, SeekOrigin.Begin); while (journal.Position < journal.Length) { // read page bytes from journal file journal.Read(buffer, 0, BasePage.PAGE_SIZE); // read pageID (first 4 bytes) var pageID = BitConverter.ToUInt32(buffer, 0); _log.Write(Logger.RECOVERY, "recover page #{0:0000}", pageID); // if header, read all byte (to get original filesize) if (pageID == 0) { var header = (HeaderPage)BasePage.ReadPage(buffer); fileSize = (header.LastPageID + 1) * BasePage.PAGE_SIZE; } // write in stream this.WritePage(pageID, buffer); } _log.Write(Logger.RECOVERY, "resize datafile to {0} bytes", fileSize); // redim filesize if grow more than original before rollback _stream.SetLength(fileSize); }
/// <summary> /// Dump all pages into a string - debug purpose only /// </summary> public StringBuilder DumpPages(uint startPage = 0, uint endPage = uint.MaxValue) { var sb = new StringBuilder(); sb.AppendLine("Dump database"); sb.AppendLine("============="); sb.AppendLine(); var header = (HeaderPage)BasePage.ReadPage(_disk.ReadPage(0)); for (uint i = startPage; i <= endPage; i++) { if (i > header.LastPageID) { break; } var p = BasePage.ReadPage(_disk.ReadPage(i)); sb.AppendFormat("{0} <{1},{2}> [{3}] {4}{5} | ", p.PageID.Dump(), p.PrevPageID.Dump(), p.NextPageID.Dump(), p.PageType.ToString().PadRight(6).Substring(0, 6), p.FreeBytes.ToString("0000"), p.IsDirty ? "d" : " "); p.Dump(sb); sb.AppendLine(); } return(sb); }
private void AvoidDirtyRead() { // if disk are exclusive don't need check dirty read if (_disk.IsExclusive) { return; } _log.Write(Logger.CACHE, "checking disk to avoid dirty read"); // empty cache? just exit if (_cache.CleanUsed == 0) { return; } // get ChangeID from cache var header = _cache.GetPage(0) as HeaderPage; var changeID = header == null ? 0 : header.ChangeID; // and get header from disk var disk = BasePage.ReadPage(_disk.ReadPage(0)) as HeaderPage; // if header change, clear cache and add new header to cache if (disk.ChangeID != changeID) { _log.Write(Logger.CACHE, "file changed from another process, cleaning all cache pages"); _cache.ClearPages(); _cache.AddPage(disk); } }
/// <summary> /// Get journal pages and override all into datafile /// </summary> public void Recovery() { _log.Write(Logger.RECOVERY, "initializing recovery mode"); using (_locker.Write()) { // double check in header need recovery (could be already recover from another thread) var header = BasePage.ReadPage(_disk.ReadPage(0)) as HeaderPage; if (header.Recovery == false) { return; } // read all journal pages foreach (var buffer in _disk.ReadJournal(header.LastPageID)) { // read pageID (first 4 bytes) var pageID = BitConverter.ToUInt32(buffer, 0); _log.Write(Logger.RECOVERY, "recover page #{0:0000}", pageID); // write in stream (encrypt if datafile is encrypted) _disk.WritePage(pageID, _crypto == null || pageID == 0 ? buffer : _crypto.Encrypt(buffer)); } // shrink datafile _disk.ClearJournal(header.LastPageID); } }
/// <summary> /// Get a page from cache or from disk (get from cache or from disk) /// </summary> public T GetPage <T>(uint pageID) where T : BasePage { lock (_disk) { var page = _cache.GetPage(pageID); // is not on cache? load from disk if (page == null) { var buffer = _disk.ReadPage(pageID); // if datafile are encrypted, decrypt buffer (header are not encrypted) if (_crypto != null && pageID > 0) { buffer = _crypto.Decrypt(buffer); } page = BasePage.ReadPage(buffer); _cache.AddPage(page); } return((T)page); } }
/// <summary> /// Get a page from cache or from disk (and put on cache) /// </summary> public T GetPage <T>(uint pageID) where T : BasePage { // lock concurrency access (read access are not in a lock transaction) lock (_cache) { var page = _cache.GetOrDefault(pageID); // is not on cache? load from disk if (page == null) { var buffer = _disk.ReadPage(pageID); // if datafile are encrypted, decrypt buffer (header are not encrypted) if (_crypto != null && pageID > 0) { buffer = _crypto.Decrypt(buffer); } page = BasePage.ReadPage(buffer); _cache.Add(pageID, page); } return((T)page); } }
/// <summary> /// Initialize LiteEngine using custom disk service implementation and full engine options /// </summary> public LiteEngine(IDiskService disk, string password = null, TimeSpan?timeout = null, int cacheSize = 5000, Logger log = null, bool utcDate = false) { if (disk == null) { throw new ArgumentNullException("disk"); } _timeout = timeout ?? TimeSpan.FromMinutes(1); _cacheSize = cacheSize; _disk = disk; _log = log ?? new Logger(); _bsonReader = new BsonReader(utcDate); try { // initialize datafile (create) and set log instance _disk.Initialize(_log, password); // lock disk (read mode) before read header var position = _disk.Lock(LockState.Read, _timeout); var buffer = _disk.ReadPage(0); _disk.Unlock(LockState.Read, position); // create header instance from array bytes var header = BasePage.ReadPage(buffer) as HeaderPage; // hash password with sha1 or keep as empty byte[20] var sha1 = password == null ? new byte[20] : AesEncryption.HashSHA1(password); // compare header password with user password even if not passed password (datafile can have password) if (sha1.BinaryCompareTo(header.Password) != 0) { throw LiteException.DatabaseWrongPassword(); } // initialize AES encryptor if (password != null) { _crypto = new AesEncryption(password, header.Salt); } // initialize all services this.InitializeServices(); // if header are marked with recovery, do it now if (header.Recovery) { _trans.Recovery(); } } catch (Exception) { // explicit dispose this.Dispose(); throw; } }
/// <summary> /// Reduce disk size re-arranging unused spaces. Can change password. If temporary disk was not provided, use MemoryStream temp disk /// </summary> public long Shrink(string password = null, IDiskService temp = null) { var originalSize = _disk.FileLength; // if temp disk are not passed, use memory stream disk temp = temp ?? new StreamDiskService(new MemoryStream()); using (_locker.Reserved()) using (_locker.Exclusive()) using (var engine = new LiteEngine(temp, password)) { // read all collection foreach (var collectionName in this.GetCollectionNames()) { // first create all user indexes (exclude _id index) foreach (var index in this.GetIndexes(collectionName).Where(x => x.Field != "_id")) { engine.EnsureIndex(collectionName, index.Field, index.Unique); } // now copy documents var docs = this.Find(collectionName, Query.All()); engine.InsertBulk(collectionName, docs); } // copy user version engine.UserVersion = this.UserVersion; // set current disk size to exact new disk usage _disk.SetLength(temp.FileLength); // read new header page to start copy var header = BasePage.ReadPage(temp.ReadPage(0)) as HeaderPage; // copy (as is) all pages from temp disk to original disk for (uint i = 0; i <= header.LastPageID; i++) { var page = temp.ReadPage(i); _disk.WritePage(i, page); } // create/destroy crypto class _crypto = password == null ? null : new AesEncryption(password, header.Salt); // initialize all services again (crypto can be changed) this.InitializeServices(); // return how many bytes are reduced return(originalSize - temp.FileLength); } }
/// <summary> /// Initialize LiteEngine using custom disk service implementation and full engine options /// </summary> public LiteEngine(IDiskService disk, string password = null, TimeSpan?timeout = null, int cacheSize = 5000, Logger log = null) { if (disk == null) { throw new ArgumentNullException("disk"); } _timeout = timeout ?? TimeSpan.FromMinutes(1); _cacheSize = cacheSize; _disk = disk; _log = log ?? new Logger(); try { // initialize datafile (create) and set log instance _disk.Initialize(_log, password); // read header page var header = BasePage.ReadPage(_disk.ReadPage(0)) as HeaderPage; // hash password with sha1 or keep as empty byte[20] var sha1 = password == null ? new byte[20] : AesEncryption.HashSHA1(password); // compare header password with user password even if not passed password (datafile can have password) if (sha1.BinaryCompareTo(header.Password) != 0) { throw LiteException.DatabaseWrongPassword(); } // initialize AES encryptor if (password != null) { _crypto = new AesEncryption(password, header.Salt); } // initialize all services this.InitializeServices(); // try recovery if has journal file _trans.Recovery(); } catch (Exception) { // explicit dispose this.Dispose(); throw; } }
/// <summary> /// Dump all pages into a string - debug purpose only /// </summary> public StringBuilder DumpPages(uint startPage = 0, uint endPage = uint.MaxValue) { var sb = new StringBuilder(); sb.AppendLine("Dump database"); sb.AppendLine("============="); sb.AppendLine(); using (var trans = _transaction.Begin(true)) { try { var header = (HeaderPage)BasePage.ReadPage(_disk.ReadPage(0)); for (uint i = startPage; i <= endPage; i++) { if (i > header.LastPageID) { break; } var p = BasePage.ReadPage(_disk.ReadPage(i)); sb.AppendFormat("{0} <{1},{2}> [{3}] {4}{5} | ", p.PageID.Dump(), p.PrevPageID.Dump(), p.NextPageID.Dump(), p.PageType.ToString().PadRight(6).Substring(0, 6), p.FreeBytes.ToString("0000"), p.IsDirty ? "d" : " "); p.Dump(sb); sb.AppendLine(); trans.Commit(); } } catch (Exception ex) { _log.Write(Logger.ERROR, ex.Message); trans.Rollback(); throw; } } return(sb); }
/// <summary> /// Test if cache still valid (if datafile was changed by another process reset cache) /// Returns true if file was changed /// [Thread Safe] /// </summary> private bool DetectDatabaseChanges() { // if disk are exclusive don't need check dirty read if (_disk.IsExclusive) { return(false); } // empty cache? just exit if (_cache.CleanUsed == 0) { return(false); } _log.Write(Logger.CACHE, "checking disk to detect database changes from another process"); // get ChangeID from cache var header = _cache.GetPage(0) as HeaderPage; var changeID = header == null ? 0 : header.ChangeID; // and get header from disk var disk = BasePage.ReadPage(_disk.ReadPage(0)) as HeaderPage; // if disk header are in recovery mode, throw exception to datafile re-open and recovery pages if (disk.Recovery) { _log.Write(Logger.ERROR, "datafile in recovery mode, need re-open database"); throw LiteException.NeedRecover(); } // if header change, clear cache and add new header to cache if (disk.ChangeID != changeID) { _log.Write(Logger.CACHE, "file changed from another process, cleaning all cache pages"); _cache.ClearPages(); _cache.AddPage(disk); return(true); } return(false); }
private static Dictionary <uint, string> RecoveryCollectionPages(LiteEngine engine, HeaderPage header, StringBuilder log) { var result = new Dictionary <uint, string>(); // get collection page foreach (var col in header.CollectionPages) { CollectionPage colPage = null; try { // read collection page var buffer = engine._disk.ReadPage(col.Value); var page = BasePage.ReadPage(buffer); if (page.PageType != PageType.Collection) { continue; } colPage = page as CollectionPage; } catch (Exception ex) { log.AppendLine($"Page {col.Value} (Collection) Error: {ex.Message}"); continue; } // get all pageID from all valid indexes var pagesID = new HashSet <uint>(colPage.Indexes.Where(x => x.IsEmpty == false && x.HeadNode.PageID != uint.MaxValue).Select(x => x.HeadNode.PageID)); // load all dataPages from this initial index pageIDs var dataPages = RecoveryDetectCollectionByIndexPages(engine, pagesID, log); // populate resultset with this collection name/data page foreach (var page in dataPages) { result[page] = col.Key; } } return(result); }
/// <summary> /// Override read page decrypting data from disk /// </summary> public override byte[] ReadPage(uint pageID) { var buffer = base.ReadPage(pageID); // when read header, checks passoword if (pageID == 0) { // I know, header page will be double read (it's the price for isolated concerns) var header = (HeaderPage)BasePage.ReadPage(buffer); if (header.DbParams.Password.BinaryCompareTo(_password) != 0) { throw LiteException.DatabaseWrongPassword(); } return(buffer); } return(_crypto.Decrypt(buffer)); }
/// <summary> /// Try recovery data from current datafile into a new datafile. /// </summary> public static string Recovery(string filename) { // if not exists, just exit if (!File.Exists(filename)) { return(""); } var log = new StringBuilder(); var newfilename = FileHelper.GetTempFile(filename, "-recovery", true); var count = 0; using (var olddb = new LiteEngine(filename)) using (var newdb = new LiteEngine(newfilename, false)) { // get header from old database (this must must be possible to read) var header = olddb._pager.GetPage <HeaderPage>(0); var collections = RecoveryCollectionPages(olddb, header, log); // try recovery all data pages for (uint i = 1; i < header.LastPageID; i++) { DataPage dataPage = null; try { var buffer = olddb._disk.ReadPage(i); // searching only for DataPage (PageType == 4) if (buffer[4] != 4) { continue; } dataPage = BasePage.ReadPage(buffer) as DataPage; } catch (Exception ex) { log.AppendLine($"Page {i} (DataPage) Error: {ex.Message}"); continue; } // try find collectionName using pageID map (use fixed name if not found) if (collections.TryGetValue(i, out var colname) == false) { colname = "_recovery"; } foreach (var block in dataPage.DataBlocks) { try { // read bytes var bson = olddb._data.Read(block.Value.Position); // deserialize as document var doc = BsonSerializer.Deserialize(bson); // and insert into new database newdb.Insert(colname, doc); count++; } catch (Exception ex) { log.AppendLine($"Document {block.Value.Position} Error: {ex.Message}"); continue; } } } } log.Insert(0, $"Document recovery count: {count}\n"); return(log.ToString()); }
private static HashSet <uint> RecoveryDetectCollectionByIndexPages(LiteEngine engine, HashSet <uint> initialPagesID, StringBuilder log) { var indexPages = new Dictionary <uint, bool>(); var dataPages = new HashSet <uint>(); foreach (var pageID in initialPagesID) { indexPages.Add(pageID, false); } // discover all indexes pages related with this current indexPage (all of them are in same collection) while (indexPages.Count(x => x.Value == false) > 0) { var item = indexPages.First(x => x.Value == false); // mark page as readed indexPages[item.Key] = true; IndexPage indexPage = null; try { // try read page from disk and deserialize as IndexPage var buffer = engine._disk.ReadPage(item.Key); var page = BasePage.ReadPage(buffer); if (page.PageType != PageType.Index) { continue; } indexPage = page as IndexPage; } catch (Exception ex) { log.AppendLine($"Page {item.Key} (Collection) Error: {ex.Message}"); continue; } // now, check for all nodes to get dataPages foreach (var node in indexPage.Nodes.Values) { if (node.DataBlock.PageID != uint.MaxValue) { dataPages.Add(node.DataBlock.PageID); } // add into indexPages all possible indexPages if (!indexPages.ContainsKey(node.PrevNode.PageID) && node.PrevNode.PageID != uint.MaxValue) { indexPages.Add(node.PrevNode.PageID, false); } if (!indexPages.ContainsKey(node.NextNode.PageID) && node.NextNode.PageID != uint.MaxValue) { indexPages.Add(node.NextNode.PageID, false); } foreach (var pos in node.Prev.Where(x => !x.IsEmpty && x.PageID != uint.MaxValue)) { if (!indexPages.ContainsKey(pos.PageID)) { indexPages.Add(pos.PageID, false); } } foreach (var pos in node.Next.Where(x => !x.IsEmpty && x.PageID != uint.MaxValue)) { if (!indexPages.ContainsKey(pos.PageID)) { indexPages.Add(pos.PageID, false); } } } } return(dataPages); }