internal override IEnumerable<IndexNode> Run(CollectionPage col, IndexService indexer) { var left = _left.Run(col, indexer); var right = _right.Run(col, indexer); return left.Union(right, new IndexNodeComparer()); }
internal override IEnumerable<IndexNode> Run(CollectionPage col, IndexService indexer) { var result = _query.Run(col, indexer); var all = new QueryAll("_id", _order).Run(col, indexer); return all.Except(result, new IndexNodeComparer()); }
/// <summary> /// Drop a collection - remove all data pages + indexes pages /// </summary> public void Drop(CollectionPage col) { // add all pages to delete var pages = new HashSet<uint>(); // search for all data page and index page foreach (var index in col.GetIndexes(true)) { // get all nodes from index var nodes = _indexer.FindAll(index, Query.Ascending); foreach (var node in nodes) { // if is PK index, add dataPages if (index.Slot == 0) { pages.Add(node.DataBlock.PageID); // read datablock to check if there is any extended page var block = _data.GetBlock(node.DataBlock); if (block.ExtendPageID != uint.MaxValue) { _pager.DeletePage(block.ExtendPageID, true); } } // memory checkpoint _trans.CheckPoint(); // add index page to delete list page pages.Add(node.Position.PageID); } // remove head+tail nodes in all indexes pages.Add(index.HeadNode.PageID); pages.Add(index.TailNode.PageID); } // and now, lets delete all this pages foreach (var pageID in pages) { // delete page _pager.DeletePage(pageID); // memory checkpoint _trans.CheckPoint(); } // get header page to remove from collection list links var header = _pager.GetPage<HeaderPage>(0); header.CollectionPages.Remove(col.CollectionName); // set header as dirty after remove _pager.SetDirty(header); _pager.DeletePage(col.PageID); }
internal override IEnumerable<IndexNode> Run(CollectionPage col, IndexService indexer) { // ignore QueryAll on AND expression (in both sides) if (_left is QueryAll) return _right.Run(col, indexer); if (_right is QueryAll) return _left.Run(col, indexer); var left = _left.Run(col, indexer); var right = _right.Run(col, indexer); return left.Intersect(right, new IndexNodeComparer()); }
private void Usage(CollectionPage col, out int indexPages, out int indexFree, out int dataPages, out int extendPages, out int dataFree) { var pages = new HashSet<uint>(); indexPages = indexFree = dataPages = extendPages = dataFree = 0; // get all pages from PK index + data/extend pages foreach (var node in _indexer.FindAll(col.PK, Query.Ascending)) { if (pages.Contains(node.Position.PageID)) continue; pages.Add(node.Position.PageID); indexPages++; indexFree += node.Page.FreeBytes; foreach (var n in node.Page.Nodes.Values.Where(x => !x.DataBlock.IsEmpty)) { var dataPage = _pager.GetPage<DataPage>(n.DataBlock.PageID, false); if(pages.Contains(dataPage.PageID)) continue; pages.Add(dataPage.PageID); dataPages++; dataFree += dataPage.FreeBytes; // getting extended pages foreach(var ex in dataPage.DataBlocks.Values.Where(x => x.ExtendPageID != uint.MaxValue)) { foreach(var extendPage in _pager.GetSeqPages<ExtendPage>(ex.ExtendPageID)) { extendPages++; dataFree += extendPage.FreeBytes; } } } _cache.CheckPoint(); } // add all others indexes foreach(var index in col.GetIndexes(false)) { foreach (var node in _indexer.FindAll(index, Query.Ascending)) { if (pages.Contains(node.Position.PageID)) continue; pages.Add(node.Position.PageID); indexPages++; indexFree += node.Page.FreeBytes; _cache.CheckPoint(); } } }
/// <summary> /// Update data inside a datapage. If new data can be used in same datapage, just update. Otherside, copy content to a new ExtendedPage /// </summary> public DataBlock Update(CollectionPage col, PageAddress blockAddress, byte[] data) { var dataPage = _pager.GetPage<DataPage>(blockAddress.PageID); var block = dataPage.DataBlocks[blockAddress.Index]; var extend = dataPage.FreeBytes + block.Data.Length - data.Length <= 0; // check if need to extend if (extend) { // clear my block data block.Data = new byte[0]; block.ExtendData = data; // create (or get a existed) extendpage and store data there ExtendPage extendPage; if (block.ExtendPageID == uint.MaxValue) { extendPage = _pager.NewPage<ExtendPage>(); block.ExtendPageID = extendPage.PageID; } else { extendPage = _pager.GetPage<ExtendPage>(block.ExtendPageID); } this.StoreExtendData(extendPage, data); } else { // if no extends, just update data block block.Data = data; // if there was a extended bytes, delete if (block.ExtendPageID != uint.MaxValue) { _pager.DeletePage(block.ExtendPageID, true); block.ExtendPageID = uint.MaxValue; } } // updates freebytes + items count dataPage.UpdateItemCount(); // add/remove dataPage on freelist if has space AND its on/off free list _pager.AddOrRemoveToFreeList(dataPage.FreeBytes > DataPage.DATA_RESERVED_BYTES, dataPage, col, ref col.FreeDataPageID); dataPage.IsDirty = true; return block; }
/// <summary> /// Internal implementation of insert a document /// </summary> private void InsertDocument(CollectionPage col, BsonDocument doc) { BsonValue id; // if no _id, add one as ObjectId if (!doc.RawValue.TryGetValue("_id", out id)) { doc["_id"] = id = ObjectId.NewObjectId(); } // test if _id is a valid type if (id.IsNull || id.IsMinValue || id.IsMaxValue) { throw LiteException.InvalidDataType("_id", id); } _log.Write(Logger.COMMAND, "insert document on '{0}' :: _id = {1}", col.CollectionName, id); // serialize object var bytes = BsonSerializer.Serialize(doc); // storage in data pages - returns dataBlock address var dataBlock = _data.Insert(col, bytes); // store id in a PK index [0 array] var pk = _indexer.AddNode(col.PK, id, null); // do link between index <-> data block pk.DataBlock = dataBlock.Position; // for each index, insert new IndexNode foreach (var index in col.GetIndexes(false)) { // for each index, get all keys (support now multi-key) - gets distinct values only // if index are unique, get single key only var keys = doc.GetValues(index.Field, index.Unique); // do a loop with all keys (multi-key supported) foreach(var key in keys) { // insert node var node = _indexer.AddNode(index, key, pk); // link my index node to data block address node.DataBlock = dataBlock.Position; } } }
/// <summary> /// Insert data inside a datapage. Returns dataPageID that idicates the first page /// </summary> public DataBlock Insert(CollectionPage col, byte[] data) { // need to extend (data is bigger than 1 page) var extend = (data.Length + DataBlock.DATA_BLOCK_FIXED_SIZE) > BasePage.PAGE_AVAILABLE_BYTES; // if extend, just search for a page with BLOCK_SIZE avaiable var dataPage = _pager.GetFreePage<DataPage>(col.FreeDataPageID, extend ? DataBlock.DATA_BLOCK_FIXED_SIZE : data.Length + DataBlock.DATA_BLOCK_FIXED_SIZE); // create a new block with first empty index on DataPage var block = new DataBlock { Position = new PageAddress(dataPage.PageID, dataPage.DataBlocks.NextIndex()), Page = dataPage }; // if extend, store all bytes on extended page. if (extend) { var extendPage = _pager.NewPage<ExtendPage>(); block.ExtendPageID = extendPage.PageID; this.StoreExtendData(extendPage, data); } else { block.Data = data; } // add dataBlock to this page dataPage.DataBlocks.Add(block.Position.Index, block); // update freebytes + items count dataPage.UpdateItemCount(); // set page as dirty _pager.SetDirty(dataPage); // add/remove dataPage on freelist if has space _pager.AddOrRemoveToFreeList(dataPage.FreeBytes > DataPage.DATA_RESERVED_BYTES, dataPage, col, ref col.FreeDataPageID); // increase document count in collection col.DocumentCount++; // set collection page as dirty _pager.SetDirty(col); return block; }
/// <summary> /// Drop a collection - remove all data pages + indexes pages /// </summary> public void Drop(CollectionPage col) { // add all pages to delete var pages = new HashSet<uint>(); // search for all data page and index page foreach (var index in col.GetIndexes(true)) { // get all nodes from index var nodes = _indexer.FindAll(index, Query.Query.Ascending); foreach (var node in nodes) { // if is PK index, add dataPages if(index.Slot == 0) { pages.Add(node.DataBlock.PageID); // read datablock to check if there is any extended page var block = _data.Read(node.DataBlock, false); if (block.ExtendPageID != uint.MaxValue) { _pager.DeletePage(block.ExtendPageID, true); } } // add index page to delete list page pages.Add(node.Position.PageID); } } // and now, lets delete all this pages foreach (var pageID in pages) { _pager.DeletePage(pageID); } // remove page from collection list _pager.AddOrRemoveToFreeList(false, col, _cache.Header, ref _cache.Header.FirstCollectionPageID); _pager.DeletePage(col.PageID, false); }
/// <summary> /// Delete one dataBlock /// </summary> public DataBlock Delete(CollectionPage col, PageAddress blockAddress) { // get page and mark as dirty var page = _pager.GetPage<DataPage>(blockAddress.PageID); var block = page.DataBlocks[blockAddress.Index]; // if there a extended page, delete all if (block.ExtendPageID != uint.MaxValue) { _pager.DeletePage(block.ExtendPageID, true); } // delete block inside page page.DataBlocks.Remove(block.Position.Index); // update freebytes + itemcount page.UpdateItemCount(); // set page as dirty here _pager.SetDirty(page); // if there is no more datablocks, lets delete all page if (page.DataBlocks.Count == 0) { // first, remove from free list _pager.AddOrRemoveToFreeList(false, page, col, ref col.FreeDataPageID); _pager.DeletePage(page.PageID); } else { // add or remove to free list _pager.AddOrRemoveToFreeList(page.FreeBytes > DataPage.DATA_RESERVED_BYTES, page, col, ref col.FreeDataPageID); } col.DocumentCount--; // mark collection page as dirty _pager.SetDirty(col); return block; }
/// <summary> /// Create a new index and returns head page address (skip list) /// </summary> public CollectionIndex CreateIndex(CollectionPage col) { // get index slot var index = col.GetFreeIndex(); // get a new index page for first index page var page = _pager.NewPage<IndexPage>(); // create a empty node with full max level var head = new IndexNode(IndexNode.MAX_LEVEL_LENGTH) { Key = BsonValue.MinValue, KeyLength = (ushort)BsonValue.MinValue.GetBytesCount(false), Slot = (byte)index.Slot, Page = page, Position = new PageAddress(page.PageID, 0) }; // add as first node page.Nodes.Add(head.Position.Index, head); // update freebytes + item count (for head) page.UpdateItemCount(); // set index page as dirty _pager.SetDirty(index.Page); // add indexPage on freelist if has space _pager.AddOrRemoveToFreeList(true, page, index.Page, ref index.FreeIndexPageID); // point the head/tail node to this new node position index.HeadNode = head.Position; // insert tail node var tail = this.AddNode(index, BsonValue.MaxValue, IndexNode.MAX_LEVEL_LENGTH, null); index.TailNode = tail.Position; return index; }
/// <summary> /// Create a new index and returns head page address (skip list) /// </summary> public CollectionIndex CreateIndex(CollectionPage col) { // get index slot var index = col.GetFreeIndex(); // get a new index page for first index page var page = _pager.NewPage <IndexPage>(); // create a empty node with full max level var head = new IndexNode(IndexNode.MAX_LEVEL_LENGTH) { Key = BsonValue.MinValue, KeyLength = (ushort)BsonValue.MinValue.GetBytesCount(false), Slot = (byte)index.Slot, Page = page, Position = new PageAddress(page.PageID, 0) }; // add as first node page.Nodes.Add(head.Position.Index, head); // update freebytes + item count (for head) page.UpdateItemCount(); // set index page as dirty _pager.SetDirty(index.Page); // add indexPage on freelist if has space _pager.AddOrRemoveToFreeList(true, page, index.Page, ref index.FreeIndexPageID); // point the head/tail node to this new node position index.HeadNode = head.Position; // insert tail node var tail = this.AddNode(index, BsonValue.MaxValue, IndexNode.MAX_LEVEL_LENGTH, null); index.TailNode = tail.Position; return(index); }
/// <summary> /// Delete one dataBlock /// </summary> public DataBlock Delete(CollectionPage col, PageAddress blockAddress) { // get page and mark as dirty var page = _pager.GetPage <DataPage>(blockAddress.PageID, true); var block = page.DataBlocks[blockAddress.Index]; // mark collection page as dirty _pager.SetDirty(col); // if there a extended page, delete all if (block.ExtendPageID != uint.MaxValue) { _pager.DeletePage(block.ExtendPageID, true); } // delete block inside page page.DataBlocks.Remove(block.Position.Index); // update freebytes + itemcount page.UpdateItemCount(); // if there is no more datablocks, lets delete the page if (page.DataBlocks.Count == 0) { // first, remove from free list _pager.AddOrRemoveToFreeList(false, page, col, ref col.FreeDataPageID); _pager.DeletePage(page.PageID); } else { // add or remove to free list _pager.AddOrRemoveToFreeList(page.FreeBytes > DataPage.DATA_RESERVED_BYTES, page, col, ref col.FreeDataPageID); } col.DocumentCount--; return(block); }
/// <summary> /// Insert data inside a datapage. Returns dataPageID that idicates the first page /// </summary> public DataBlock Insert(CollectionPage col, IndexKey key, byte[] data) { // need to extend (data is bigger than 1 page) var extend = (data.Length + key.Length + DataBlock.DATA_BLOCK_FIXED_SIZE) > BasePage.PAGE_AVAILABLE_BYTES; // if extend, just search for a page with BLOCK_SIZE avaiable var dataPage = _pager.GetFreePage <DataPage>(col.FreeDataPageID, extend ? DataBlock.DATA_BLOCK_FIXED_SIZE : key.Length + data.Length + DataBlock.DATA_BLOCK_FIXED_SIZE); // create a new block with first empty index on DataPage var block = new DataBlock { Position = new PageAddress(dataPage.PageID, dataPage.DataBlocks.NextIndex()), Page = dataPage, Key = key }; // if extend, store all bytes on extended page. if (extend) { var extendPage = _pager.NewPage <ExtendPage>(); block.ExtendPageID = extendPage.PageID; this.StoreExtendData(extendPage, data); } else { block.Data = data; } // add dataBlock to this page dataPage.DataBlocks.Add(block.Position.Index, block); dataPage.IsDirty = true; // add/remove dataPage on freelist if has space _pager.AddOrRemoveToFreeList(dataPage.FreeBytes > BasePage.RESERVED_BYTES, dataPage, col, ref col.FreeDataPageID); col.DocumentCount++; col.IsDirty = true; return(block); }
/// <summary> /// Find witch index will be used and run Execute method /// </summary> internal virtual IEnumerable<IndexNode> Run(CollectionPage col, IndexService indexer) { // get index for this query var index = col.GetIndex(this.Field); // no index? throw an index not found exception to auto-create in LiteDatabse if (index == null) throw new IndexNotFoundException(col.CollectionName, this.Field); // execute query to get all IndexNodes return this.ExecuteIndex(indexer, index); }
/// <summary> /// Implement internal update document /// </summary> private bool UpdateDocument(CollectionPage col, BsonDocument doc) { // normalize id before find var id = doc["_id"]; // validate id for null, min/max values if (id.IsNull || id.IsMinValue || id.IsMaxValue) { throw LiteException.InvalidDataType("_id", id); } _log.Write(Logger.COMMAND, "update document on '{0}' :: _id = {1}", col.CollectionName, id); // find indexNode from pk index var pkNode = _indexer.Find(col.PK, id, false, Query.Ascending); // if not found document, no updates if (pkNode == null) return false; // serialize document in bytes var bytes = BsonSerializer.Serialize(doc); // update data storage var dataBlock = _data.Update(col, pkNode.DataBlock, bytes); // get all non-pk index nodes from this data block var allNodes = _indexer.GetNodeList(pkNode, false).ToArray(); // delete/insert indexes - do not touch on PK foreach (var index in col.GetIndexes(false)) { // using Distinct/ToArray because I will do many queries var keys = doc.GetValues(index.Field, true, index.Unique).ToArray(); // get a list of to delete nodes (using ToArray to resolve now) var toDelete = allNodes .Where(x => x.Slot == index.Slot && !keys.Any(k => k == x.Key)) .ToArray(); // get a list of to insert nodes (using ToArray to resolve now) var toInsert = keys .Where(x => !allNodes.Any(k => k.Slot == index.Slot && k.Key == x)) .ToArray(); // delete changed index nodes foreach (var node in toDelete) { _indexer.Delete(index, node.Position); } // insert new nodes foreach (var key in toInsert) { // and add a new one var node = _indexer.AddNode(index, key, pkNode); // link my node to data block node.DataBlock = dataBlock.Position; } } return true; }
private void Usage(CollectionPage col, out int indexPages, out int indexFree, out int dataPages, out int extendPages, out int dataFree, out int docSize) { var pages = new HashSet <uint>(); indexPages = indexFree = dataPages = extendPages = dataFree = docSize = 0; // get all pages from PK index + data/extend pages foreach (var node in _indexer.FindAll(col.PK, Query.Ascending)) { if (pages.Contains(node.Position.PageID)) { continue; } pages.Add(node.Position.PageID); indexPages++; indexFree += node.Page.FreeBytes; foreach (var n in node.Page.Nodes.Values.Where(x => !x.DataBlock.IsEmpty)) { var dataPage = _pager.GetPage <DataPage>(n.DataBlock.PageID, false); if (pages.Contains(dataPage.PageID)) { continue; } foreach (var block in dataPage.DataBlocks.Values) { var doc = BsonSerializer.Deserialize(_data.Read(block.Position, true).Buffer); docSize += doc.GetBytesCount(true); } pages.Add(dataPage.PageID); dataPages++; dataFree += dataPage.FreeBytes; // getting extended pages foreach (var ex in dataPage.DataBlocks.Values.Where(x => x.ExtendPageID != uint.MaxValue)) { foreach (var extendPage in _pager.GetSeqPages <ExtendPage>(ex.ExtendPageID)) { extendPages++; dataFree += extendPage.FreeBytes; } } } _cache.CheckPoint(); } // add all others indexes foreach (var index in col.GetIndexes(false)) { foreach (var node in _indexer.FindAll(index, Query.Ascending)) { if (pages.Contains(node.Position.PageID)) { continue; } pages.Add(node.Position.PageID); indexPages++; indexFree += node.Page.FreeBytes; _cache.CheckPoint(); } } }
/// <summary> /// Drop a collection, ignoring exceptions on pages inside this collection /// </summary> /// <param name="collectionPage"></param> public void DropForced(CollectionPage collectionPage) { try { var pagesToDrop = new HashSet <uint>(); var indexes = collectionPage.GetIndexes(true); foreach (var collectionIndex in indexes) { try { var indexNodes = _indexer.FindAll(collectionIndex, Query.Ascending); foreach (var node in indexNodes) { if (collectionIndex.Slot == 0) { pagesToDrop.Add(node.DataBlock.PageID); var block = _data.GetBlock(node.DataBlock); if (block.ExtendPageID != uint.MaxValue) { _pager.DeletePage(block.ExtendPageID, true); _trans.CheckPoint(); } } pagesToDrop.Add(node.Position.PageID); } } catch { _log.Write(Logger.ERROR, "Exception during force drop iteration"); } finally { pagesToDrop.Add(collectionIndex.HeadNode.PageID); pagesToDrop.Add(collectionIndex.TailNode.PageID); } } pagesToDrop.Add(collectionPage.PageID); foreach (var pageToDrop in pagesToDrop) { _pager.DeletePage(pageToDrop); _trans.CheckPoint(); } } catch { _log.Write(Logger.ERROR, "Exception during force drop"); } finally { var header = _pager.GetPage <HeaderPage>(0); header.CollectionPages.Remove(collectionPage.CollectionName); _pager.SetDirty(header); } }
/// <summary> /// Internal implementation of insert a document /// </summary> private void InsertDocument(CollectionPage col, BsonDocument doc, BsonType autoId) { // collection Sequence was created after release current datafile version. // In this case, Sequence will be 0 but already has documents. Let's fix this // ** this code can be removed when datafile change from 7 (HeaderPage.FILE_VERSION) ** if (col.Sequence == 0 && col.DocumentCount > 0) { var max = this.Max(col.CollectionName, "_id"); // if max value is a number, convert to Sequence last value // if not, just set sequence as document count col.Sequence = (max.IsInt32 || max.IsInt64 || max.IsDouble || max.IsDecimal) ? Convert.ToInt64(max.RawValue) : Convert.ToInt64(col.DocumentCount); } // increase collection sequence _id col.Sequence++; _pager.SetDirty(col); // if no _id, add one if (!doc.RawValue.TryGetValue("_id", out var id)) { doc["_id"] = id = autoId == BsonType.ObjectId ? new BsonValue(ObjectId.NewObjectId()) : autoId == BsonType.Guid ? new BsonValue(Guid.NewGuid()) : autoId == BsonType.DateTime ? new BsonValue(DateTime.Now) : autoId == BsonType.Int32 ? new BsonValue((Int32)col.Sequence) : autoId == BsonType.Int64 ? new BsonValue(col.Sequence) : BsonValue.Null; } // create bubble in sequence number if _id is bigger than current sequence else if (autoId == BsonType.Int32 || autoId == BsonType.Int64) { var current = id.AsInt64; // if current id is bigger than sequence, jump sequence to this number. Other was, do not increse sequnce col.Sequence = current >= col.Sequence ? current : col.Sequence - 1; } // test if _id is a valid type if (id.IsNull || id.IsMinValue || id.IsMaxValue) { throw LiteException.InvalidDataType("_id", id); } _log.Write(Logger.COMMAND, "insert document on '{0}' :: _id = {1}", col.CollectionName, id.RawValue); // serialize object var bytes = _bsonWriter.Serialize(doc); // storage in data pages - returns dataBlock address var dataBlock = _data.Insert(col, bytes); // store id in a PK index [0 array] var pk = _indexer.AddNode(col.PK, id, null); // do link between index <-> data block pk.DataBlock = dataBlock.Position; // for each index, insert new IndexNode foreach (var index in col.GetIndexes(false)) { // for each index, get all keys (support now multi-key) - gets distinct values only // if index are unique, get single key only var expr = new BsonExpression(index.Expression); var keys = expr.Execute(doc, true); // do a loop with all keys (multi-key supported) foreach (var key in keys) { // insert node var node = _indexer.AddNode(index, key, pk); // link my index node to data block address node.DataBlock = dataBlock.Position; } } }
/// <summary> /// Implement internal update document /// </summary> private bool UpdateDocument(CollectionPage col, BsonDocument doc) { // normalize id before find var id = doc["_id"]; // validate id for null, min/max values if (id.IsNull || id.IsMinValue || id.IsMaxValue) { throw LiteException.InvalidDataType("_id", id); } _log.Write(Logger.COMMAND, "update document on '{0}' :: _id = {1}", col.CollectionName, id); // find indexNode from pk index var pkNode = _indexer.Find(col.PK, id, false, Query.Ascending); // if not found document, no updates if (pkNode == null) { return(false); } // serialize document in bytes var bytes = BsonSerializer.Serialize(doc); // update data storage var dataBlock = _data.Update(col, pkNode.DataBlock, bytes); // get all non-pk index nodes from this data block var allNodes = _indexer.GetNodeList(pkNode, false).ToArray(); // delete/insert indexes - do not touch on PK foreach (var index in col.GetIndexes(false)) { // using Distinct/ToArray because I will do many queries var keys = doc.GetValues(index.Field, true, index.Unique).ToArray(); // get a list of to delete nodes (using ToArray to resolve now) var toDelete = allNodes .Where(x => x.Slot == index.Slot && !keys.Any(k => k == x.Key)) .ToArray(); // get a list of to insert nodes (using ToArray to resolve now) var toInsert = keys .Where(x => !allNodes.Any(k => k.Slot == index.Slot && k.Key == x)) .ToArray(); // delete changed index nodes foreach (var node in toDelete) { _indexer.Delete(index, node.Position); } // insert new nodes foreach (var key in toInsert) { // and add a new one var node = _indexer.AddNode(index, key, pkNode); // link my node to data block node.DataBlock = dataBlock.Position; } } return(true); }