/// <summary>
/// Find the field inside document tree, using json-like path, and update with value paramter. If field nod exists, create new field. Return true if document was changed
/// </summary>
public bool Set(string path, BsonValue value)
{
if (string.IsNullOrEmpty(path))
{
throw new ArgumentNullException(nameof(value));
}
if (value == null)
{
throw new ArgumentNullException(nameof(value));
}
var field = path.StartsWith("$") ? path : "$." + path;
var parent = field.Substring(0, field.LastIndexOf('.'));
var key = field.Substring(field.LastIndexOf('.') + 1);
var expr = new BsonExpression(parent);
var changed = false;
foreach (var item in expr.Execute(this, false).Where(x => x.IsDocument))
{
var idoc = item.AsDocument;
var cur = idoc[key];
// update field only if value are different from current value
if (cur != value)
{
idoc[key] = value;
changed = true;
}
}
return(changed);
}
/// <summary>
/// Set or add a value to document using a json-like path to update/create this field. If you addInArray, only add if path returns an array.
/// </summary>
public bool Set(string path, BsonValue value, bool addInArray)
{
if (string.IsNullOrEmpty(path))
{
throw new ArgumentNullException(nameof(value));
}
if (value == null)
{
throw new ArgumentNullException(nameof(value));
}
if (addInArray == false)
{
return(this.Set(path, value));
}
var expr = new BsonExpression(path.StartsWith("$") ? path : "$." + path);
var changed = false;
foreach (var arr in expr.Execute(this, false).Where(x => x.IsArray))
{
arr.AsArray.Add(value);
changed = true;
}
return(changed);
}
private void InsertSet(string path, object value)
{
_updates.Add((doc) =>
{
var field = path.StartsWith("$") ? path : "$." + path;
var parent = field.Substring(0, field.LastIndexOf('.'));
var key = field.Substring(field.LastIndexOf('.') + 1);
var expr = new BsonExpression(parent);
var val = value is BsonValue ? value as BsonValue : (value as BsonExpression).Execute(doc, true).First();
var changed = false;
foreach (var item in expr.Execute(doc, false).Where(x => x.IsDocument))
{
var idoc = item.AsDocument;
var cur = idoc[key];
// update field only if value are different from current value
if (cur != val)
{
idoc[key] = val;
changed = true;
}
}
return(changed);
});
}
/// <summary>
/// Find for documents in a collection using Query definition. Support for include reference documents. Use Path syntax
/// </summary>
public IEnumerable <BsonDocument> Find(string collection, Query query, string[] includes, int skip = 0, int limit = int.MaxValue)
{
if (includes == null)
{
throw new ArgumentNullException(nameof(includes));
}
var docs = this.Find(collection, query, skip, limit);
foreach (var doc in docs)
{
// procced with all includes
foreach (var include in includes)
{
var expr = new BsonExpression(include.StartsWith("$") ? include : "$." + include);
// get all values according JSON path
foreach (var value in expr.Execute(doc, false)
.Where(x => x.IsDocument)
.Select(x => x.AsDocument)
.ToList())
{
// works only if is a document
var refId = value["$id"];
var refCol = value["$ref"];
// if has no reference, just go out
if (refId.IsNull || !refCol.IsString)
{
continue;
}
// now, find document reference
var refDoc = this.FindById(refCol, refId);
// if found, change with current document
if (refDoc != null)
{
value.Remove("$id");
value.Remove("$ref");
refDoc.CopyTo(value);
}
else
{
// remove value from parent (document or array)
value.Destroy();
}
}
}
yield return(doc);
}
}
/// <summary>
/// Find the field inside document tree, using json-like path, and update with an expression paramter. If field nod exists, create new field. Return true if document was changed
/// </summary>
public bool Set(string path, BsonExpression expr)
{
if (expr == null)
{
throw new ArgumentNullException(nameof(expr));
}
var value = expr.Execute(this, true).First();
return(this.Set(path, value));
}
/// <summary>
/// Returns all values from array according index. If index are MaxValue, return all values
/// </summary>
public static IEnumerable <BsonValue> Array(IEnumerable <BsonValue> values, int index, BsonExpression expr, BsonDocument root)
{
foreach (var value in values)
{
if (value.IsArray)
{
var arr = value.AsArray;
// [<expr>] - index are an expression
if (expr.Source != null)
{
foreach (var item in arr)
{
// execute for each child value and except a first bool value (returns if true)
var c = expr.Execute(root, item, true).First();
if (c.IsBoolean && c.AsBoolean == true)
{
// fill destroy action to remove value from parent array
item.Destroy = () => arr.Remove(item);
yield return(item);
}
}
}
// [*] - index are all values
else if (index == int.MaxValue)
{
foreach (var item in arr)
{
// fill destroy action to remove value from parent array
item.Destroy = () => arr.Remove(item);
yield return(item);
}
}
// [n] - fixed index
else
{
var idx = index < 0 ? arr.Count + index : index;
if (arr.Count > idx)
{
var item = arr[idx];
// fill destroy action to remove value from parent array
item.Destroy = () => arr.Remove(item);
yield return(item);
}
}
}
}
}
public static IEnumerable <BsonValue> MAP(BsonDocument root, Collation collation, BsonDocument parameters, IEnumerable <BsonValue> input, BsonExpression mapExpr)
{
foreach (var item in input)
{
// execute for each child value and except a first bool value (returns if true)
var values = mapExpr.Execute(new BsonDocument[] { root }, root, item, collation);
foreach (var value in values)
{
yield return(value);
}
}
}
public static IEnumerable <BsonValue> MAP(IEnumerable <BsonValue> input, BsonExpression mapExpr, BsonDocument root, BsonDocument parameters)
{
// update parameters in expression
parameters.CopyTo(mapExpr.Parameters);
foreach (var item in input)
{
// execute for each child value and except a first bool value (returns if true)
var values = mapExpr.Execute(new BsonDocument[] { root }, root, item);
foreach (var value in values)
{
yield return(value);
}
}
}
private void InsertAdd(string path, object value)
{
_updates.Add((doc) =>
{
var expr = new BsonExpression(path.StartsWith("$") ? path : "$." + path);
var val = value is BsonValue ? value as BsonValue : (value as BsonExpression).Execute(doc, true).First();
var changed = false;
foreach (var arr in expr.Execute(doc, false).Where(x => x.IsArray))
{
arr.AsArray.Add(val);
changed = true;
}
return(changed);
});
}
/// <summary>
/// Returns all values from array according index. If index are MaxValue, return all values
/// </summary>
public static IEnumerable <BsonValue> Array(IEnumerable <BsonValue> values, int index, string expr, BsonDocument root)
{
foreach (var value in values)
{
if (value.IsArray)
{
var arr = value.AsArray;
// [expression(Func<BsonValue, bool>)]
if (expr != "")
{
var e = new BsonExpression(expr, false);
foreach (var item in arr)
{
// execute for each child value and except a first bool value (returns if true)
var c = e.Execute(root, item, true).First();
if (c.IsBoolean && c.AsBoolean == true)
{
yield return(item);
}
}
}
// [all]
else if (index == int.MaxValue)
{
foreach (var item in arr)
{
yield return(item);
}
}
// [fixed_index]
else
{
var idx = index < 0 ? arr.Count + index : index;
if (arr.Count > idx)
{
yield return(arr[idx]);
}
}
}
}
}
/// <summary>
/// Create a new index (or do nothing if already exists) to a collection/field
/// </summary>
public bool EnsureIndex(string collection, string field, bool unique = false, string expression = null)
{
if (collection.IsNullOrWhiteSpace())
{
throw new ArgumentNullException("collection");
}
if (!CollectionIndex.IndexPattern.IsMatch(field))
{
throw new ArgumentException("Invalid field format pattern: " + CollectionIndex.IndexPattern.ToString(), "field");
}
if (field == "_id")
{
return(false); // always exists
}
if (expression != null && expression.Length > 200)
{
throw new ArgumentException("expression is limited in 200 characters", "expression");
}
return(this.Transaction <bool>(collection, true, (col) =>
{
// check if index already exists
var current = col.GetIndex(field);
// if already exists, just exit
if (current != null)
{
// do not test any difference between current index and new defition
return false;
}
// create index head
var index = _indexer.CreateIndex(col);
index.Field = field;
index.Expression = expression ?? "$." + field;
index.Unique = unique;
_log.Write(Logger.COMMAND, "create index on '{0}' :: {1} unique: {2}", collection, index.Expression, unique);
// read all objects (read from PK index)
foreach (var pkNode in new QueryAll("_id", Query.Ascending).Run(col, _indexer))
{
// read binary and deserialize document
var buffer = _data.Read(pkNode.DataBlock);
var doc = BsonSerializer.Deserialize(buffer).AsDocument;
var expr = new BsonExpression(index.Expression);
// get values from expression in document
var keys = expr.Execute(doc, true);
// adding index node for each value
foreach (var key in keys)
{
// insert new index node
var node = _indexer.AddNode(index, key, pkNode);
// link index node to datablock
node.DataBlock = pkNode.DataBlock;
}
// check memory usage
_trans.CheckPoint();
}
return true;
}));
}
/// <summary>
/// Internal implementation of insert a document
/// </summary>
private void InsertDocument(CollectionPage col, BsonDocument doc, BsonType autoId)
{
// collection Sequence was created after release current datafile version.
// In this case, Sequence will be 0 but already has documents. Let's fix this
// ** this code can be removed when datafile change from 7 (HeaderPage.FILE_VERSION) **
if (col.Sequence == 0 && col.DocumentCount > 0)
{
var max = this.Max(col.CollectionName, "_id");
// if max value is a number, convert to Sequence last value
// if not, just set sequence as document count
col.Sequence = (max.IsInt32 || max.IsInt64 || max.IsDouble || max.IsDecimal) ?
Convert.ToInt64(max.RawValue) :
Convert.ToInt64(col.DocumentCount);
}
// increase collection sequence _id
col.Sequence++;
_pager.SetDirty(col);
// if no _id, add one
if (!doc.RawValue.TryGetValue("_id", out var id))
{
doc["_id"] = id =
autoId == BsonType.ObjectId ? new BsonValue(ObjectId.NewObjectId()) :
autoId == BsonType.Guid ? new BsonValue(Guid.NewGuid()) :
autoId == BsonType.DateTime ? new BsonValue(DateTime.Now) :
autoId == BsonType.Int32 ? new BsonValue((Int32)col.Sequence) :
autoId == BsonType.Int64 ? new BsonValue(col.Sequence) : BsonValue.Null;
}
// create bubble in sequence number if _id is bigger than current sequence
else if (autoId == BsonType.Int32 || autoId == BsonType.Int64)
{
var current = id.AsInt64;
// if current id is bigger than sequence, jump sequence to this number. Other was, do not increse sequnce
col.Sequence = current >= col.Sequence ? current : col.Sequence - 1;
}
// test if _id is a valid type
if (id.IsNull || id.IsMinValue || id.IsMaxValue)
{
throw LiteException.InvalidDataType("_id", id);
}
_log.Write(Logger.COMMAND, "insert document on '{0}' :: _id = {1}", col.CollectionName, id.RawValue);
// serialize object
var bytes = _bsonWriter.Serialize(doc);
// storage in data pages - returns dataBlock address
var dataBlock = _data.Insert(col, bytes);
// store id in a PK index [0 array]
var pk = _indexer.AddNode(col.PK, id, null);
// do link between index <-> data block
pk.DataBlock = dataBlock.Position;
// for each index, insert new IndexNode
foreach (var index in col.GetIndexes(false))
{
// for each index, get all keys (support now multi-key) - gets distinct values only
// if index are unique, get single key only
var expr = new BsonExpression(index.Expression);
var keys = expr.Execute(doc, true);
// do a loop with all keys (multi-key supported)
foreach (var key in keys)
{
// insert node
var node = _indexer.AddNode(index, key, pk);
// link my index node to data block address
node.DataBlock = dataBlock.Position;
}
}
}
/// <summary>
/// Implement internal update document
/// </summary>
private bool UpdateDocument(CollectionPage col, BsonDocument doc)
{
// normalize id before find
var id = doc["_id"];
// validate id for null, min/max values
if (id.IsNull || id.IsMinValue || id.IsMaxValue)
{
throw LiteException.InvalidDataType("_id", id);
}
_log.Write(Logger.COMMAND, "update document on '{0}' :: _id = {1}", col.CollectionName, id.RawValue);
// find indexNode from pk index
var pkNode = _indexer.Find(col.PK, id, false, Query.Ascending);
// if not found document, no updates
if (pkNode == null)
{
return(false);
}
// serialize document in bytes
var bytes = _bsonWriter.Serialize(doc);
// update data storage
var dataBlock = _data.Update(col, pkNode.DataBlock, bytes);
// get all non-pk index nodes from this data block
var allNodes = _indexer.GetNodeList(pkNode, false).ToArray();
// delete/insert indexes - do not touch on PK
foreach (var index in col.GetIndexes(false))
{
var expr = new BsonExpression(index.Expression);
// getting all keys do check
var keys = expr.Execute(doc).ToArray();
// get a list of to delete nodes (using ToArray to resolve now)
var toDelete = allNodes
.Where(x => x.Slot == index.Slot && !keys.Any(k => k == x.Key))
.ToArray();
// get a list of to insert nodes (using ToArray to resolve now)
var toInsert = keys
.Where(x => !allNodes.Any(k => k.Slot == index.Slot && k.Key == x))
.ToArray();
// delete changed index nodes
foreach (var node in toDelete)
{
_indexer.Delete(index, node.Position);
}
// insert new nodes
foreach (var key in toInsert)
{
// and add a new one
var node = _indexer.AddNode(index, key, pkNode);
// link my node to data block
node.DataBlock = dataBlock.Position;
}
}
return(true);
}
/// <summary>
/// Get an IEnumerable of values from a json-like path inside document. Use BsonExpression to parse this path
/// </summary>
public IEnumerable <BsonValue> Get(string path, bool includeNullIfEmpty = false)
{
var expr = new BsonExpression(new StringScanner(path), true, true);
return(expr.Execute(this, includeNullIfEmpty));
}
private const int MAX_SORT_PAGES = 5000; // ~ 20Mb?
/// <summary>
/// EXPERIMENTAL Find with sort operation - use memory or disk (temp file) to sort
/// </summary>
public List <BsonDocument> FindSort(string collection, Query query, string orderBy, int order = Query.Ascending, int skip = 0, int limit = int.MaxValue)
{
if (collection.IsNullOrWhiteSpace())
{
throw new ArgumentNullException(nameof(collection));
}
if (query == null)
{
throw new ArgumentNullException(nameof(query));
}
_log.Write(Logger.COMMAND, "query-sort documents in '{0}' => {1}", collection, query);
// evaluate orderBy path/expression
var expr = new BsonExpression(orderBy);
// lock database for read access
using (_locker.Read())
{
var last = order == Query.Ascending ? BsonValue.MaxValue : BsonValue.MinValue;
var total = limit == int.MaxValue ? int.MaxValue : skip + limit;
var indexCounter = 0;
var disk = new TempDiskService();
// create memory database
using (var engine = new LiteEngine(disk))
{
// get collection page
var col = this.GetCollectionPage(collection, false);
if (col == null)
{
return(new List <BsonDocument>());
}
// create a temp collection in new memory database
var tmp = engine._collections.Add("tmp");
// create index pointer
var index = engine._indexer.CreateIndex(tmp);
// get head/tail index node
var head = engine._indexer.GetNode(index.HeadNode);
var tail = engine._indexer.GetNode(index.TailNode);
// first lets works only with index in query
var nodes = query.Run(col, _indexer);
foreach (var node in nodes)
{
var buffer = _data.Read(node.DataBlock);
var doc = _bsonReader.Deserialize(buffer).AsDocument;
// if needs use filter
if (query.UseFilter && query.FilterDocument(doc) == false)
{
continue;
}
// get key to be sorted
var key = expr.Execute(doc, true).First();
var diff = key.CompareTo(last);
// add to list only if lower than last space
if ((order == Query.Ascending && diff < 1) ||
(order == Query.Descending && diff > -1))
{
var tmpNode = engine._indexer.AddNode(index, key, null);
tmpNode.DataBlock = node.DataBlock;
tmpNode.CacheDocument = doc;
indexCounter++;
// exceeded limit
if (indexCounter > total)
{
var exceeded = (order == Query.Ascending) ? tail.Prev[0] : head.Next[0];
engine._indexer.Delete(index, exceeded);
var lnode = (order == Query.Ascending) ? tail.Prev[0] : head.Next[0];
last = engine._indexer.GetNode(lnode).Key;
indexCounter--;
}
// if memory pages excedded limit size, flush to disk
if (engine._cache.DirtyUsed > MAX_SORT_PAGES)
{
engine._trans.PersistDirtyPages();
engine._trans.CheckPoint();
}
}
}
var result = new List <BsonDocument>();
// if skip is lower than limit, take nodes from skip from begin
// if skip is higher than limit, take nodes from end and revert order (avoid lots of skip)
var find = skip < limit?
engine._indexer.FindAll(index, order).Skip(skip).Take(limit) : // get from original order
engine._indexer.FindAll(index, -order).Take(limit).Reverse(); // avoid long skips, take from end and revert
// --- foreach (var node in engine._indexer.FindAll(index, order).Skip(skip).Take(limit))
foreach (var node in find)
{
// if document are in cache, use it. if not, get from disk again
var doc = node.CacheDocument;
if (doc == null)
{
var buffer = _data.Read(node.DataBlock);
doc = _bsonReader.Deserialize(buffer).AsDocument;
}
result.Add(doc);
}
return(result);
}
}
}
