/// <summary>
/// Create a new index and returns head page address (skip list)
/// </summary>
public CollectionIndex CreateIndex(string name, string expr, bool unique)
{
// get how many butes needed fore each head/tail (both has same size)
var bytesLength = IndexNode.GetNodeLength(MAX_LEVEL_LENGTH, BsonValue.MinValue, out var keyLength);
// get a new empty page (each index contains your own linked nodes)
var indexPage = _snapshot.NewPage <IndexPage>();
// create index ref
var index = _snapshot.CollectionPage.InsertCollectionIndex(name, expr, unique);
// insert head/tail nodes
var head = indexPage.InsertIndexNode(index.Slot, MAX_LEVEL_LENGTH, BsonValue.MinValue, PageAddress.Empty, bytesLength);
var tail = indexPage.InsertIndexNode(index.Slot, MAX_LEVEL_LENGTH, BsonValue.MaxValue, PageAddress.Empty, bytesLength);
// link head-to-tail with double link list in first level
head.SetNext(0, tail.Position);
tail.SetPrev(0, head.Position);
// add this new page in free list (slot 0)
index.FreeIndexPageList = indexPage.PageID;
indexPage.PageListSlot = 0;
index.Head = head.Position;
index.Tail = tail.Position;
return(index);
}
/// <summary>
/// Loop in values enumerator to return N values for a single container
/// </summary>
private IEnumerable <KeyValuePair <BsonValue, PageAddress> > YieldValues(IEnumerator <KeyValuePair <BsonValue, PageAddress> > source, Done done)
{
var size = IndexNode.GetKeyLength(source.Current.Key, false) + PageAddress.SIZE;
if (size > MAX_INDEX_KEY_LENGTH)
{
throw new LiteException(0, $"Current value are larger than {MAX_INDEX_KEY_LENGTH} bytes and can't be sorted.");
}
yield return(source.Current);
while (source.MoveNext())
{
var length = IndexNode.GetKeyLength(source.Current.Key, false) + PageAddress.SIZE;
done.Count++;
if (size + length > _containerSize)
{
yield break;
}
size += length;
yield return(source.Current);
}
done.Running = false;
}
/// <summary>
/// Delete a single index node - fix tree double-linked list levels
/// </summary>
private void DeleteSingleNode(IndexNode node)
{
for (int i = node.Level - 1; i >= 0; i--)
{
// get previous and next nodes (between my deleted node)
var prevNode = this.GetNode(node.Prev[i]);
var nextNode = this.GetNode(node.Next[i]);
if (prevNode != null)
{
prevNode.SetNext((byte)i, node.Next[i]);
}
if (nextNode != null)
{
nextNode.SetPrev((byte)i, node.Prev[i]);
}
}
// get current slot position in free list
var slot = BasePage.FreeIndexSlot(node.Page.FreeBytes);
node.Page.DeleteIndexNode(node.Position.Index);
// update (if needed) slot position
_snapshot.AddOrRemoveFreeList(node.Page, slot);
}
/// <summary>
/// Read single IndexNode
/// </summary>
public IndexNode GetIndexNode(byte index)
{
var segment = base.Get(index);
var node = new IndexNode(this, index, segment);
return(node);
}
/// <summary>
/// Insert new IndexNode. After call this, "node" instance can't be changed
/// </summary>
public IndexNode InsertIndexNode(byte slot, byte level, BsonValue key, PageAddress dataBlock, int bytesLength)
{
var segment = base.Insert((ushort)bytesLength, out var index);
var node = new IndexNode(this, index, segment, slot, level, key, dataBlock);
return(node);
}
public BsonDocument Load(IndexNode node)
{
ENSURE(node.DataBlock.IsEmpty == false, "Never should be empty rawid");
var doc = new BsonDocument
{
[_name] = node.Key,
};
doc.RawId = node.DataBlock;
return(doc);
}
/// <summary>
/// Read single IndexNode
/// </summary>
public IndexNode GetIndexNode(byte index)
{
//if (_cache.TryGetValue(index, out var node))
//{
// return node;
//}
//else
{
var segment = base.Get(index);
var node = new IndexNode(this, index, segment);
//_cache[index] = node;
return(node);
}
}
/// <summary>
/// Internal implementation of insert a document
/// </summary>
private void InsertDocument(Snapshot snapshot, BsonDocument doc, BsonAutoId autoId, IndexService indexer, DataService data)
{
// if no _id, use AutoId
if (!doc.TryGetValue("_id", out var id))
{
doc["_id"] = id =
autoId == BsonAutoId.ObjectId ? new BsonValue(ObjectId.NewObjectId()) :
autoId == BsonAutoId.Guid ? new BsonValue(Guid.NewGuid()) :
this.GetSequence(snapshot, autoId);
}
else if (id.IsNumber)
{
// update memory sequence of numeric _id
this.SetSequence(snapshot, id);
}
// test if _id is a valid type
if (id.IsNull || id.IsMinValue || id.IsMaxValue)
{
throw LiteException.InvalidDataType("_id", id);
}
// storage in data pages - returns dataBlock address
var dataBlock = data.Insert(doc);
IndexNode last = null;
// for each index, insert new IndexNode
foreach (var index in snapshot.CollectionPage.GetCollectionIndexes())
{
// for each index, get all keys (supports multi-key) - gets distinct values only
// if index are unique, get single key only
var keys = index.BsonExpr.Execute(doc, _header.Pragmas.Collation);
// do a loop with all keys (multi-key supported)
foreach (var key in keys)
{
// insert node
var node = indexer.AddNode(index, key, dataBlock, last);
last = node;
}
}
}
/// <summary>
/// Delete a single index node - fix tree double-linked list levels
/// </summary>
private void DeleteSingleNode(IndexNode node, CollectionIndex index)
{
for (int i = node.Level - 1; i >= 0; i--)
{
// get previous and next nodes (between my deleted node)
var prevNode = this.GetNode(node.Prev[i]);
var nextNode = this.GetNode(node.Next[i]);
if (prevNode != null)
{
prevNode.SetNext((byte)i, node.Next[i]);
}
if (nextNode != null)
{
nextNode.SetPrev((byte)i, node.Prev[i]);
}
}
node.Page.DeleteIndexNode(node.Position.Index);
_snapshot.AddOrRemoveFreeIndexList(node.Page, ref index.FreeIndexPageList);
}
/// <summary>
/// Loop in values enumerator to return N values for a single container
/// </summary>
private IEnumerable <KeyValuePair <BsonValue, PageAddress> > YieldValues(IEnumerator <KeyValuePair <BsonValue, PageAddress> > source, Done done)
{
var size = IndexNode.GetKeyLength(source.Current.Key, false) + PageAddress.SIZE;
yield return(source.Current);
while (source.MoveNext())
{
var length = IndexNode.GetKeyLength(source.Current.Key, false) + PageAddress.SIZE;
done.Count++;
if (size + length > _containerSize)
{
yield break;
}
size += length;
yield return(source.Current);
}
done.Running = false;
}
/// <summary>
/// Create a new index and returns head page address (skip list)
/// </summary>
public CollectionIndex CreateIndex(string name, string expr, bool unique)
{
// get how many butes needed fore each head/tail (both has same size)
var bytesLength = IndexNode.GetNodeLength(MAX_LEVEL_LENGTH, BsonValue.MinValue, out var keyLength);
// get a free index page for head note (x2 for head + tail)
var indexPage = _snapshot.GetFreePage <IndexPage>(bytesLength * 2);
// create index ref
var index = _snapshot.CollectionPage.InsertCollectionIndex(name, expr, unique);
// insert head/tail nodes
var head = indexPage.InsertIndexNode(index.Slot, MAX_LEVEL_LENGTH, BsonValue.MinValue, PageAddress.Empty, bytesLength);
var tail = indexPage.InsertIndexNode(index.Slot, MAX_LEVEL_LENGTH, BsonValue.MaxValue, PageAddress.Empty, bytesLength);
// link head-to-tail with double link list in first level
head.SetNext(0, tail.Position);
tail.SetPrev(0, head.Position);
index.Head = head.Position;
index.Tail = tail.Position;
return(index);
}
public virtual BsonDocument Load(IndexNode node)
{
ENSURE(node.DataBlock != PageAddress.Empty, "data block must be a valid block address");
return(this.Load(node.DataBlock));
}
/// <summary>
/// Create a new index (or do nothing if already exists) to a collection/field
/// </summary>
public bool EnsureIndex(string collection, string name, BsonExpression expression, bool unique)
{
if (collection.IsNullOrWhiteSpace())
{
throw new ArgumentNullException(nameof(collection));
}
if (name.IsNullOrWhiteSpace())
{
throw new ArgumentNullException(nameof(name));
}
if (expression == null)
{
throw new ArgumentNullException(nameof(expression));
}
if (expression.IsIndexable == false)
{
throw new ArgumentException("Index expressions must contains at least one document field. Used methods must be immutable. Parameters are not supported.", nameof(expression));
}
if (name.Length > INDEX_NAME_MAX_LENGTH)
{
throw LiteException.InvalidIndexName(name, collection, "MaxLength = " + INDEX_NAME_MAX_LENGTH);
}
if (!name.IsWord())
{
throw LiteException.InvalidIndexName(name, collection, "Use only [a-Z$_]");
}
if (name.StartsWith("$"))
{
throw LiteException.InvalidIndexName(name, collection, "Index name can't starts with `$`");
}
if (name == "_id")
{
return(false); // always exists
}
return(this.AutoTransaction(transaction =>
{
var snapshot = transaction.CreateSnapshot(LockMode.Write, collection, true);
var col = snapshot.CollectionPage;
var indexer = new IndexService(snapshot);
var data = new DataService(snapshot);
// check if index already exists
var current = col.GetCollectionIndex(name);
// if already exists, just exit
if (current != null)
{
// but if expression are different, throw error
if (current.Expression != expression.Source)
{
throw LiteException.IndexAlreadyExist(name);
}
return false;
}
LOG($"create index `{collection}.{name}`", "COMMAND");
// create index head
var index = indexer.CreateIndex(name, expression.Source, unique);
var count = 0u;
// read all objects (read from PK index)
foreach (var pkNode in new IndexAll("_id", LiteDB.Query.Ascending).Run(col, indexer))
{
using (var reader = new BufferReader(data.Read(pkNode.DataBlock)))
{
var doc = reader.ReadDocument(expression.Fields);
// first/last node in this document that will be added
IndexNode last = null;
IndexNode first = null;
// get values from expression in document
var keys = expression.Execute(doc);
// adding index node for each value
foreach (var key in keys)
{
// when index key is an array, get items inside array.
// valid only for first level (if this items are another array, this arrays will be indexed as array)
if (key.IsArray)
{
var arr = key.AsArray;
foreach (var itemKey in arr)
{
// insert new index node
var node = indexer.AddNode(index, itemKey, pkNode.DataBlock, last, _flipCoin);
if (first == null)
{
first = node;
}
last = node;
count++;
}
}
else
{
// insert new index node
var node = indexer.AddNode(index, key, pkNode.DataBlock, last, _flipCoin);
if (first == null)
{
first = node;
}
last = node;
count++;
}
}
// fix single linked-list in pkNode
if (first != null)
{
last.SetNextNode(pkNode.NextNode);
pkNode.SetNextNode(first.Position);
}
}
transaction.Safepoint();
}
index.KeyCount = count;
return true;
}));
}
/// <summary>
/// Insert a new node index inside an collection index.
/// </summary>
private IndexNode AddNode(CollectionIndex index, BsonValue key, PageAddress dataBlock, byte level, IndexNode last)
{
// get a free index page for head note
var bytesLength = IndexNode.GetNodeLength(level, key, out var keyLength);
// test for index key maxlength
if (keyLength > MAX_INDEX_KEY_LENGTH)
{
throw LiteException.InvalidIndexKey($"Index key must be less than {MAX_INDEX_KEY_LENGTH} bytes.");
}
var indexPage = _snapshot.GetFreeIndexPage(bytesLength, ref index.FreeIndexPageList);
// create node in buffer
var node = indexPage.InsertIndexNode(index.Slot, level, key, dataBlock, bytesLength);
// now, let's link my index node on right place
var cur = this.GetNode(index.Head);
// using as cache last
IndexNode cache = null;
// scan from top left
for (int i = index.MaxLevel - 1; i >= 0; i--)
{
// get cache for last node
cache = cache != null && cache.Position == cur.Next[i] ? cache : this.GetNode(cur.Next[i]);
// for(; <while_not_this>; <do_this>) { ... }
for (; cur.Next[i].IsEmpty == false; cur = cache)
{
// get cache for last node
cache = cache != null && cache.Position == cur.Next[i] ? cache : this.GetNode(cur.Next[i]);
// read next node to compare
var diff = cache.Key.CompareTo(key, _collation);
// if unique and diff = 0, throw index exception (must rollback transaction - others nodes can be dirty)
if (diff == 0 && index.Unique)
{
throw LiteException.IndexDuplicateKey(index.Name, key);
}
if (diff == 1)
{
break;
}
}
if (i <= (level - 1)) // level == length
{
// cur = current (immediately before - prev)
// node = new inserted node
// next = next node (where cur is pointing)
node.SetNext((byte)i, cur.Next[i]);
node.SetPrev((byte)i, cur.Position);
cur.SetNext((byte)i, node.Position);
var next = this.GetNode(node.Next[i]);
if (next != null)
{
next.SetPrev((byte)i, node.Position);
}
}
}
// if last node exists, create a double link list
if (last != null)
{
ENSURE(last.NextNode == PageAddress.Empty, "last index node must point to null");
last.SetNextNode(node.Position);
}
// fix page position in free list slot
_snapshot.AddOrRemoveFreeIndexList(node.Page, ref index.FreeIndexPageList);
return(node);
}
/// <summary>
/// Create a new index (or do nothing if already exists) to a collection/field
/// </summary>
public async Task <bool> EnsureIndexAsync(string collection, string name, BsonExpression expression, bool unique)
{
if (collection.IsNullOrWhiteSpace())
{
throw new ArgumentNullException(nameof(collection));
}
if (name.IsNullOrWhiteSpace())
{
throw new ArgumentNullException(nameof(name));
}
if (expression == null)
{
throw new ArgumentNullException(nameof(expression));
}
if (expression.IsIndexable == false)
{
throw new ArgumentException("Index expressions must contains at least one document field. Used methods must be immutable. Parameters are not supported.", nameof(expression));
}
if (name.Length > INDEX_NAME_MAX_LENGTH)
{
throw LiteException.InvalidIndexName(name, collection, "MaxLength = " + INDEX_NAME_MAX_LENGTH);
}
if (!name.IsWord())
{
throw LiteException.InvalidIndexName(name, collection, "Use only [a-Z$_]");
}
if (name.StartsWith("$"))
{
throw LiteException.InvalidIndexName(name, collection, "Index name can't starts with `$`");
}
if (expression.IsScalar == false && unique)
{
throw new LiteException(0, "Multikey index expression do not support unique option");
}
if (expression.Source == "$._id")
{
return(false); // always exists
}
return(await this.AutoTransaction(async transaction =>
{
var snapshot = await transaction.CreateSnapshot(LockMode.Write, collection, true);
var collectionPage = snapshot.CollectionPage;
var indexer = new IndexService(snapshot, _header.Pragmas.Collation);
var data = new DataService(snapshot);
// check if index already exists
var current = collectionPage.GetCollectionIndex(name);
// if already exists, just exit
if (current != null)
{
// but if expression are different, throw error
if (current.Expression != expression.Source)
{
throw LiteException.IndexAlreadyExist(name);
}
return false;
}
LOG($"create index `{collection}.{name}`", "COMMAND");
// create index head
var index = await indexer.CreateIndex(name, expression.Source, unique);
var count = 0u;
// read all objects (read from PK index)
await foreach (var pkNode in new IndexAll("_id", LiteDB.Query.Ascending).Run(collectionPage, indexer))
{
await using (var reader = await BufferReaderAsync.CreateAsync(data.Read(pkNode.DataBlock)))
{
var doc = await reader.ReadDocument(expression.Fields);
// first/last node in this document that will be added
IndexNode last = null;
IndexNode first = null;
// get values from expression in document
var keys = expression.GetIndexKeys(doc, _header.Pragmas.Collation);
// adding index node for each value
foreach (var key in keys)
{
// insert new index node
var node = await indexer.AddNode(index, key, pkNode.DataBlock, last);
if (first == null)
{
first = node;
}
last = node;
count++;
}
// fix single linked-list in pkNode
if (first != null)
{
last.SetNextNode(pkNode.NextNode);
pkNode.SetNextNode(first.Position);
}
}
await transaction.Safepoint();
}
return true;
}));
}
public BsonDocument Load(IndexNode node)
{
return(node.Key as BsonDocument);
}
/// <summary>
/// Insert a new node index inside an collection index. Flip coin to know level
/// </summary>
public IndexNode AddNode(CollectionIndex index, BsonValue key, PageAddress dataBlock, IndexNode last)
{
// do not accept Min/Max value as index key (only head/tail can have this value)
if (key.IsMaxValue || key.IsMinValue)
{
throw LiteException.InvalidIndexKey($"BsonValue MaxValue/MinValue are not supported as index key");
}
// random level (flip coin mode) - return number between 1-32
var level = this.Flip();
// set index collection with max-index level
if (level > index.MaxLevel)
{
// update max level
_snapshot.CollectionPage.UpdateCollectionIndex(index.Name).MaxLevel = level;
}
// call AddNode with key value
return(this.AddNode(index, key, dataBlock, level, last));
}
public Task <BsonDocument> Load(IndexNode node)
{
return(Task.FromResult(node.Key as BsonDocument));
}