예제 #1
0
        /// <summary>
        /// Create a new index and returns head page address (skip list)
        /// </summary>
        public CollectionIndex CreateIndex(string name, string expr, bool unique)
        {
            // get how many butes needed fore each head/tail (both has same size)
            var bytesLength = IndexNode.GetNodeLength(MAX_LEVEL_LENGTH, BsonValue.MinValue, out var keyLength);

            // get a new empty page (each index contains your own linked nodes)
            var indexPage = _snapshot.NewPage <IndexPage>();

            // create index ref
            var index = _snapshot.CollectionPage.InsertCollectionIndex(name, expr, unique);

            // insert head/tail nodes
            var head = indexPage.InsertIndexNode(index.Slot, MAX_LEVEL_LENGTH, BsonValue.MinValue, PageAddress.Empty, bytesLength);
            var tail = indexPage.InsertIndexNode(index.Slot, MAX_LEVEL_LENGTH, BsonValue.MaxValue, PageAddress.Empty, bytesLength);

            // link head-to-tail with double link list in first level
            head.SetNext(0, tail.Position);
            tail.SetPrev(0, head.Position);

            // add this new page in free list (slot 0)
            index.FreeIndexPageList = indexPage.PageID;
            indexPage.PageListSlot  = 0;

            index.Head = head.Position;
            index.Tail = tail.Position;

            return(index);
        }
예제 #2
0
        /// <summary>
        /// Loop in values enumerator to return N values for a single container
        /// </summary>
        private IEnumerable <KeyValuePair <BsonValue, PageAddress> > YieldValues(IEnumerator <KeyValuePair <BsonValue, PageAddress> > source, Done done)
        {
            var size = IndexNode.GetKeyLength(source.Current.Key, false) + PageAddress.SIZE;

            if (size > MAX_INDEX_KEY_LENGTH)
            {
                throw new LiteException(0, $"Current value are larger than {MAX_INDEX_KEY_LENGTH} bytes and can't be sorted.");
            }

            yield return(source.Current);

            while (source.MoveNext())
            {
                var length = IndexNode.GetKeyLength(source.Current.Key, false) + PageAddress.SIZE;

                done.Count++;

                if (size + length > _containerSize)
                {
                    yield break;
                }

                size += length;

                yield return(source.Current);
            }

            done.Running = false;
        }
예제 #3
0
        /// <summary>
        /// Delete a single index node - fix tree double-linked list levels
        /// </summary>
        private void DeleteSingleNode(IndexNode node)
        {
            for (int i = node.Level - 1; i >= 0; i--)
            {
                // get previous and next nodes (between my deleted node)
                var prevNode = this.GetNode(node.Prev[i]);
                var nextNode = this.GetNode(node.Next[i]);

                if (prevNode != null)
                {
                    prevNode.SetNext((byte)i, node.Next[i]);
                }
                if (nextNode != null)
                {
                    nextNode.SetPrev((byte)i, node.Prev[i]);
                }
            }

            // get current slot position in free list
            var slot = BasePage.FreeIndexSlot(node.Page.FreeBytes);

            node.Page.DeleteIndexNode(node.Position.Index);

            // update (if needed) slot position
            _snapshot.AddOrRemoveFreeList(node.Page, slot);
        }
예제 #4
0
        /// <summary>
        /// Read single IndexNode
        /// </summary>
        public IndexNode GetIndexNode(byte index)
        {
            var segment = base.Get(index);

            var node = new IndexNode(this, index, segment);

            return(node);
        }
예제 #5
0
        /// <summary>
        /// Insert new IndexNode. After call this, "node" instance can't be changed
        /// </summary>
        public IndexNode InsertIndexNode(byte slot, byte level, BsonValue key, PageAddress dataBlock, int bytesLength)
        {
            var segment = base.Insert((ushort)bytesLength, out var index);

            var node = new IndexNode(this, index, segment, slot, level, key, dataBlock);

            return(node);
        }
예제 #6
0
        public BsonDocument Load(IndexNode node)
        {
            ENSURE(node.DataBlock.IsEmpty == false, "Never should be empty rawid");

            var doc = new BsonDocument
            {
                [_name] = node.Key,
            };

            doc.RawId = node.DataBlock;

            return(doc);
        }
예제 #7
0
        /// <summary>
        /// Read single IndexNode
        /// </summary>
        public IndexNode GetIndexNode(byte index)
        {
            //if (_cache.TryGetValue(index, out var node))
            //{
            //    return node;
            //}
            //else
            {
                var segment = base.Get(index);

                var node = new IndexNode(this, index, segment);

                //_cache[index] = node;

                return(node);
            }
        }
예제 #8
0
파일: Insert.cs 프로젝트: yaho86/LiteDB
        /// <summary>
        /// Internal implementation of insert a document
        /// </summary>
        private void InsertDocument(Snapshot snapshot, BsonDocument doc, BsonAutoId autoId, IndexService indexer, DataService data)
        {
            // if no _id, use AutoId
            if (!doc.TryGetValue("_id", out var id))
            {
                doc["_id"] = id =
                    autoId == BsonAutoId.ObjectId ? new BsonValue(ObjectId.NewObjectId()) :
                    autoId == BsonAutoId.Guid ? new BsonValue(Guid.NewGuid()) :
                    this.GetSequence(snapshot, autoId);
            }
            else if (id.IsNumber)
            {
                // update memory sequence of numeric _id
                this.SetSequence(snapshot, id);
            }

            // test if _id is a valid type
            if (id.IsNull || id.IsMinValue || id.IsMaxValue)
            {
                throw LiteException.InvalidDataType("_id", id);
            }

            // storage in data pages - returns dataBlock address
            var dataBlock = data.Insert(doc);

            IndexNode last = null;

            // for each index, insert new IndexNode
            foreach (var index in snapshot.CollectionPage.GetCollectionIndexes())
            {
                // for each index, get all keys (supports multi-key) - gets distinct values only
                // if index are unique, get single key only
                var keys = index.BsonExpr.Execute(doc, _header.Pragmas.Collation);

                // do a loop with all keys (multi-key supported)
                foreach (var key in keys)
                {
                    // insert node
                    var node = indexer.AddNode(index, key, dataBlock, last);

                    last = node;
                }
            }
        }
예제 #9
0
        /// <summary>
        /// Delete a single index node - fix tree double-linked list levels
        /// </summary>
        private void DeleteSingleNode(IndexNode node, CollectionIndex index)
        {
            for (int i = node.Level - 1; i >= 0; i--)
            {
                // get previous and next nodes (between my deleted node)
                var prevNode = this.GetNode(node.Prev[i]);
                var nextNode = this.GetNode(node.Next[i]);

                if (prevNode != null)
                {
                    prevNode.SetNext((byte)i, node.Next[i]);
                }
                if (nextNode != null)
                {
                    nextNode.SetPrev((byte)i, node.Prev[i]);
                }
            }

            node.Page.DeleteIndexNode(node.Position.Index);

            _snapshot.AddOrRemoveFreeIndexList(node.Page, ref index.FreeIndexPageList);
        }
예제 #10
0
파일: SortService.cs 프로젝트: zamis/LiteDB
        /// <summary>
        /// Loop in values enumerator to return N values for a single container
        /// </summary>
        private IEnumerable <KeyValuePair <BsonValue, PageAddress> > YieldValues(IEnumerator <KeyValuePair <BsonValue, PageAddress> > source, Done done)
        {
            var size = IndexNode.GetKeyLength(source.Current.Key, false) + PageAddress.SIZE;

            yield return(source.Current);

            while (source.MoveNext())
            {
                var length = IndexNode.GetKeyLength(source.Current.Key, false) + PageAddress.SIZE;

                done.Count++;

                if (size + length > _containerSize)
                {
                    yield break;
                }

                size += length;

                yield return(source.Current);
            }

            done.Running = false;
        }
예제 #11
0
        /// <summary>
        /// Create a new index and returns head page address (skip list)
        /// </summary>
        public CollectionIndex CreateIndex(string name, string expr, bool unique)
        {
            // get how many butes needed fore each head/tail (both has same size)
            var bytesLength = IndexNode.GetNodeLength(MAX_LEVEL_LENGTH, BsonValue.MinValue, out var keyLength);

            // get a free index page for head note (x2 for head + tail)
            var indexPage = _snapshot.GetFreePage <IndexPage>(bytesLength * 2);

            // create index ref
            var index = _snapshot.CollectionPage.InsertCollectionIndex(name, expr, unique);

            // insert head/tail nodes
            var head = indexPage.InsertIndexNode(index.Slot, MAX_LEVEL_LENGTH, BsonValue.MinValue, PageAddress.Empty, bytesLength);
            var tail = indexPage.InsertIndexNode(index.Slot, MAX_LEVEL_LENGTH, BsonValue.MaxValue, PageAddress.Empty, bytesLength);

            // link head-to-tail with double link list in first level
            head.SetNext(0, tail.Position);
            tail.SetPrev(0, head.Position);

            index.Head = head.Position;
            index.Tail = tail.Position;

            return(index);
        }
예제 #12
0
        public virtual BsonDocument Load(IndexNode node)
        {
            ENSURE(node.DataBlock != PageAddress.Empty, "data block must be a valid block address");

            return(this.Load(node.DataBlock));
        }
예제 #13
0
        /// <summary>
        /// Create a new index (or do nothing if already exists) to a collection/field
        /// </summary>
        public bool EnsureIndex(string collection, string name, BsonExpression expression, bool unique)
        {
            if (collection.IsNullOrWhiteSpace())
            {
                throw new ArgumentNullException(nameof(collection));
            }
            if (name.IsNullOrWhiteSpace())
            {
                throw new ArgumentNullException(nameof(name));
            }
            if (expression == null)
            {
                throw new ArgumentNullException(nameof(expression));
            }
            if (expression.IsIndexable == false)
            {
                throw new ArgumentException("Index expressions must contains at least one document field. Used methods must be immutable. Parameters are not supported.", nameof(expression));
            }

            if (name.Length > INDEX_NAME_MAX_LENGTH)
            {
                throw LiteException.InvalidIndexName(name, collection, "MaxLength = " + INDEX_NAME_MAX_LENGTH);
            }
            if (!name.IsWord())
            {
                throw LiteException.InvalidIndexName(name, collection, "Use only [a-Z$_]");
            }
            if (name.StartsWith("$"))
            {
                throw LiteException.InvalidIndexName(name, collection, "Index name can't starts with `$`");
            }

            if (name == "_id")
            {
                return(false);               // always exists
            }
            return(this.AutoTransaction(transaction =>
            {
                var snapshot = transaction.CreateSnapshot(LockMode.Write, collection, true);
                var col = snapshot.CollectionPage;
                var indexer = new IndexService(snapshot);
                var data = new DataService(snapshot);

                // check if index already exists
                var current = col.GetCollectionIndex(name);

                // if already exists, just exit
                if (current != null)
                {
                    // but if expression are different, throw error
                    if (current.Expression != expression.Source)
                    {
                        throw LiteException.IndexAlreadyExist(name);
                    }

                    return false;
                }

                LOG($"create index `{collection}.{name}`", "COMMAND");

                // create index head
                var index = indexer.CreateIndex(name, expression.Source, unique);
                var count = 0u;

                // read all objects (read from PK index)
                foreach (var pkNode in new IndexAll("_id", LiteDB.Query.Ascending).Run(col, indexer))
                {
                    using (var reader = new BufferReader(data.Read(pkNode.DataBlock)))
                    {
                        var doc = reader.ReadDocument(expression.Fields);

                        // first/last node in this document that will be added
                        IndexNode last = null;
                        IndexNode first = null;

                        // get values from expression in document
                        var keys = expression.Execute(doc);

                        // adding index node for each value
                        foreach (var key in keys)
                        {
                            // when index key is an array, get items inside array.
                            // valid only for first level (if this items are another array, this arrays will be indexed as array)
                            if (key.IsArray)
                            {
                                var arr = key.AsArray;

                                foreach (var itemKey in arr)
                                {
                                    // insert new index node
                                    var node = indexer.AddNode(index, itemKey, pkNode.DataBlock, last, _flipCoin);

                                    if (first == null)
                                    {
                                        first = node;
                                    }

                                    last = node;

                                    count++;
                                }
                            }
                            else
                            {
                                // insert new index node
                                var node = indexer.AddNode(index, key, pkNode.DataBlock, last, _flipCoin);

                                if (first == null)
                                {
                                    first = node;
                                }

                                last = node;

                                count++;
                            }
                        }

                        // fix single linked-list in pkNode
                        if (first != null)
                        {
                            last.SetNextNode(pkNode.NextNode);
                            pkNode.SetNextNode(first.Position);
                        }
                    }

                    transaction.Safepoint();
                }

                index.KeyCount = count;

                return true;
            }));
        }
예제 #14
0
        /// <summary>
        /// Insert a new node index inside an collection index.
        /// </summary>
        private IndexNode AddNode(CollectionIndex index, BsonValue key, PageAddress dataBlock, byte level, IndexNode last)
        {
            // get a free index page for head note
            var bytesLength = IndexNode.GetNodeLength(level, key, out var keyLength);

            // test for index key maxlength
            if (keyLength > MAX_INDEX_KEY_LENGTH)
            {
                throw LiteException.InvalidIndexKey($"Index key must be less than {MAX_INDEX_KEY_LENGTH} bytes.");
            }

            var indexPage = _snapshot.GetFreeIndexPage(bytesLength, ref index.FreeIndexPageList);

            // create node in buffer
            var node = indexPage.InsertIndexNode(index.Slot, level, key, dataBlock, bytesLength);

            // now, let's link my index node on right place
            var cur = this.GetNode(index.Head);

            // using as cache last
            IndexNode cache = null;

            // scan from top left
            for (int i = index.MaxLevel - 1; i >= 0; i--)
            {
                // get cache for last node
                cache = cache != null && cache.Position == cur.Next[i] ? cache : this.GetNode(cur.Next[i]);

                // for(; <while_not_this>; <do_this>) { ... }
                for (; cur.Next[i].IsEmpty == false; cur = cache)
                {
                    // get cache for last node
                    cache = cache != null && cache.Position == cur.Next[i] ? cache : this.GetNode(cur.Next[i]);

                    // read next node to compare
                    var diff = cache.Key.CompareTo(key, _collation);

                    // if unique and diff = 0, throw index exception (must rollback transaction - others nodes can be dirty)
                    if (diff == 0 && index.Unique)
                    {
                        throw LiteException.IndexDuplicateKey(index.Name, key);
                    }

                    if (diff == 1)
                    {
                        break;
                    }
                }

                if (i <= (level - 1)) // level == length
                {
                    // cur = current (immediately before - prev)
                    // node = new inserted node
                    // next = next node (where cur is pointing)

                    node.SetNext((byte)i, cur.Next[i]);
                    node.SetPrev((byte)i, cur.Position);
                    cur.SetNext((byte)i, node.Position);

                    var next = this.GetNode(node.Next[i]);

                    if (next != null)
                    {
                        next.SetPrev((byte)i, node.Position);
                    }
                }
            }

            // if last node exists, create a double link list
            if (last != null)
            {
                ENSURE(last.NextNode == PageAddress.Empty, "last index node must point to null");

                last.SetNextNode(node.Position);
            }

            // fix page position in free list slot
            _snapshot.AddOrRemoveFreeIndexList(node.Page, ref index.FreeIndexPageList);

            return(node);
        }
예제 #15
0
        /// <summary>
        /// Create a new index (or do nothing if already exists) to a collection/field
        /// </summary>
        public async Task <bool> EnsureIndexAsync(string collection, string name, BsonExpression expression, bool unique)
        {
            if (collection.IsNullOrWhiteSpace())
            {
                throw new ArgumentNullException(nameof(collection));
            }
            if (name.IsNullOrWhiteSpace())
            {
                throw new ArgumentNullException(nameof(name));
            }
            if (expression == null)
            {
                throw new ArgumentNullException(nameof(expression));
            }
            if (expression.IsIndexable == false)
            {
                throw new ArgumentException("Index expressions must contains at least one document field. Used methods must be immutable. Parameters are not supported.", nameof(expression));
            }

            if (name.Length > INDEX_NAME_MAX_LENGTH)
            {
                throw LiteException.InvalidIndexName(name, collection, "MaxLength = " + INDEX_NAME_MAX_LENGTH);
            }
            if (!name.IsWord())
            {
                throw LiteException.InvalidIndexName(name, collection, "Use only [a-Z$_]");
            }
            if (name.StartsWith("$"))
            {
                throw LiteException.InvalidIndexName(name, collection, "Index name can't starts with `$`");
            }
            if (expression.IsScalar == false && unique)
            {
                throw new LiteException(0, "Multikey index expression do not support unique option");
            }

            if (expression.Source == "$._id")
            {
                return(false);                              // always exists
            }
            return(await this.AutoTransaction(async transaction =>
            {
                var snapshot = await transaction.CreateSnapshot(LockMode.Write, collection, true);
                var collectionPage = snapshot.CollectionPage;
                var indexer = new IndexService(snapshot, _header.Pragmas.Collation);
                var data = new DataService(snapshot);

                // check if index already exists
                var current = collectionPage.GetCollectionIndex(name);

                // if already exists, just exit
                if (current != null)
                {
                    // but if expression are different, throw error
                    if (current.Expression != expression.Source)
                    {
                        throw LiteException.IndexAlreadyExist(name);
                    }

                    return false;
                }

                LOG($"create index `{collection}.{name}`", "COMMAND");

                // create index head
                var index = await indexer.CreateIndex(name, expression.Source, unique);
                var count = 0u;

                // read all objects (read from PK index)
                await foreach (var pkNode in new IndexAll("_id", LiteDB.Query.Ascending).Run(collectionPage, indexer))
                {
                    await using (var reader = await BufferReaderAsync.CreateAsync(data.Read(pkNode.DataBlock)))
                    {
                        var doc = await reader.ReadDocument(expression.Fields);

                        // first/last node in this document that will be added
                        IndexNode last = null;
                        IndexNode first = null;

                        // get values from expression in document
                        var keys = expression.GetIndexKeys(doc, _header.Pragmas.Collation);

                        // adding index node for each value
                        foreach (var key in keys)
                        {
                            // insert new index node
                            var node = await indexer.AddNode(index, key, pkNode.DataBlock, last);

                            if (first == null)
                            {
                                first = node;
                            }

                            last = node;

                            count++;
                        }

                        // fix single linked-list in pkNode
                        if (first != null)
                        {
                            last.SetNextNode(pkNode.NextNode);
                            pkNode.SetNextNode(first.Position);
                        }
                    }

                    await transaction.Safepoint();
                }

                return true;
            }));
        }
예제 #16
0
 public BsonDocument Load(IndexNode node)
 {
     return(node.Key as BsonDocument);
 }
예제 #17
0
        /// <summary>
        /// Insert a new node index inside an collection index. Flip coin to know level
        /// </summary>
        public IndexNode AddNode(CollectionIndex index, BsonValue key, PageAddress dataBlock, IndexNode last)
        {
            // do not accept Min/Max value as index key (only head/tail can have this value)
            if (key.IsMaxValue || key.IsMinValue)
            {
                throw LiteException.InvalidIndexKey($"BsonValue MaxValue/MinValue are not supported as index key");
            }

            // random level (flip coin mode) - return number between 1-32
            var level = this.Flip();

            // set index collection with max-index level
            if (level > index.MaxLevel)
            {
                // update max level
                _snapshot.CollectionPage.UpdateCollectionIndex(index.Name).MaxLevel = level;
            }

            // call AddNode with key value
            return(this.AddNode(index, key, dataBlock, level, last));
        }
예제 #18
0
 public Task <BsonDocument> Load(IndexNode node)
 {
     return(Task.FromResult(node.Key as BsonDocument));
 }