/// <summary> /// Set or add a value to document using a json-like path to update/create this field. If you addInArray, only add if path returns an array. /// </summary> public bool Set(string path, BsonValue value, bool addInArray) { if (string.IsNullOrEmpty(path)) { throw new ArgumentNullException(nameof(value)); } if (value == null) { throw new ArgumentNullException(nameof(value)); } if (addInArray == false) { return(this.Set(path, value)); } var expr = new BsonExpression(path.StartsWith("$") ? path : "$." + path); var changed = false; foreach (var arr in expr.Execute(this, false).Where(x => x.IsArray)) { arr.AsArray.Add(value); changed = true; } return(changed); }
/// <summary> /// Find the field inside document tree, using json-like path, and update with value paramter. If field nod exists, create new field. Return true if document was changed /// </summary> public bool Set(string path, BsonValue value) { if (string.IsNullOrEmpty(path)) { throw new ArgumentNullException(nameof(value)); } if (value == null) { throw new ArgumentNullException(nameof(value)); } var field = path.StartsWith("$") ? path : "$." + path; var parent = field.Substring(0, field.LastIndexOf('.')); var key = field.Substring(field.LastIndexOf('.') + 1); var expr = new BsonExpression(parent); var changed = false; foreach (var item in expr.Execute(this, false).Where(x => x.IsDocument)) { var idoc = item.AsDocument; var cur = idoc[key]; // update field only if value are different from current value if (cur != value) { idoc[key] = value; changed = true; } } return(changed); }
/// <summary> /// Find for documents in a collection using Query definition. Support for include reference documents. Use Path syntax /// </summary> public IEnumerable <BsonDocument> Find(string collection, Query query, string[] includes, int skip = 0, int limit = int.MaxValue) { if (includes == null) { throw new ArgumentNullException(nameof(includes)); } var docs = this.Find(collection, query, skip, limit); foreach (var doc in docs) { // procced with all includes foreach (var include in includes) { var expr = new BsonExpression(include.StartsWith("$") ? include : "$." + include); // get all values according JSON path foreach (var value in expr.Execute(doc, false) .Where(x => x.IsDocument) .Select(x => x.AsDocument) .ToList()) { // works only if is a document var refId = value["$id"]; var refCol = value["$ref"]; // if has no reference, just go out if (refId.IsNull || !refCol.IsString) { continue; } // now, find document reference var refDoc = this.FindById(refCol, refId); // if found, change with current document if (refDoc != null) { value.Remove("$id"); value.Remove("$ref"); refDoc.CopyTo(value); } else { // remove value from parent (document or array) value.Destroy(); } } } yield return(doc); } }
/// <summary> /// Returns all values from array according index. If index are MaxValue, return all values /// </summary> public static IEnumerable <BsonValue> Array(IEnumerable <BsonValue> values, int index, BsonExpression expr, BsonDocument root) { foreach (var value in values) { if (value.IsArray) { var arr = value.AsArray; // [<expr>] - index are an expression if (expr.Source != null) { foreach (var item in arr) { // execute for each child value and except a first bool value (returns if true) var c = expr.Execute(root, item, true).First(); if (c.IsBoolean && c.AsBoolean == true) { // fill destroy action to remove value from parent array item.Destroy = () => arr.Remove(item); yield return(item); } } } // [*] - index are all values else if (index == int.MaxValue) { foreach (var item in arr) { // fill destroy action to remove value from parent array item.Destroy = () => arr.Remove(item); yield return(item); } } // [n] - fixed index else { var idx = index < 0 ? arr.Count + index : index; if (arr.Count > idx) { var item = arr[idx]; // fill destroy action to remove value from parent array item.Destroy = () => arr.Remove(item); yield return(item); } } } } }
/// <summary> /// Find the field inside document tree, using json-like path, and update with an expression paramter. If field nod exists, create new field. Return true if document was changed /// </summary> public bool Set(string path, BsonExpression expr) { if (expr == null) { throw new ArgumentNullException(nameof(expr)); } var value = expr.Execute(this, true).First(); return(this.Set(path, value)); }
/// <summary> /// Internal implementation of insert a document /// </summary> private void InsertDocument(CollectionPage col, BsonDocument doc, BsonType autoId) { // collection Sequence was created after release current datafile version. // In this case, Sequence will be 0 but already has documents. Let's fix this // ** this code can be removed when datafile change from 7 (HeaderPage.FILE_VERSION) ** if (col.Sequence == 0 && col.DocumentCount > 0) { var max = this.Max(col.CollectionName, "_id"); // if max value is a number, convert to Sequence last value // if not, just set sequence as document count col.Sequence = (max.IsInt32 || max.IsInt64 || max.IsDouble || max.IsDecimal) ? Convert.ToInt64(max.RawValue) : Convert.ToInt64(col.DocumentCount); } // increase collection sequence _id col.Sequence++; _pager.SetDirty(col); // if no _id, add one if (!doc.RawValue.TryGetValue("_id", out var id)) { doc["_id"] = id = autoId == BsonType.ObjectId ? new BsonValue(ObjectId.NewObjectId()) : autoId == BsonType.Guid ? new BsonValue(Guid.NewGuid()) : autoId == BsonType.DateTime ? new BsonValue(DateTime.Now) : autoId == BsonType.Int32 ? new BsonValue((Int32)col.Sequence) : autoId == BsonType.Int64 ? new BsonValue(col.Sequence) : BsonValue.Null; } // create bubble in sequence number if _id is bigger than current sequence else if (autoId == BsonType.Int32 || autoId == BsonType.Int64) { var current = id.AsInt64; // if current id is bigger than sequence, jump sequence to this number. Other was, do not increse sequnce col.Sequence = current >= col.Sequence ? current : col.Sequence - 1; } // test if _id is a valid type if (id.IsNull || id.IsMinValue || id.IsMaxValue) { throw LiteException.InvalidDataType("_id", id); } _log.Write(Logger.COMMAND, "insert document on '{0}' :: _id = {1}", col.CollectionName, id.RawValue); // serialize object var bytes = _bsonWriter.Serialize(doc); // storage in data pages - returns dataBlock address var dataBlock = _data.Insert(col, bytes); // store id in a PK index [0 array] var pk = _indexer.AddNode(col.PK, id, null); // do link between index <-> data block pk.DataBlock = dataBlock.Position; // for each index, insert new IndexNode foreach (var index in col.GetIndexes(false)) { // for each index, get all keys (support now multi-key) - gets distinct values only // if index are unique, get single key only var expr = new BsonExpression(index.Expression); var keys = expr.Execute(doc, true); // do a loop with all keys (multi-key supported) foreach (var key in keys) { // insert node var node = _indexer.AddNode(index, key, pk); // link my index node to data block address node.DataBlock = dataBlock.Position; } } }
/// <summary> /// Create a new index (or do nothing if already exists) to a collection/field /// </summary> public bool EnsureIndex(string collection, string field, string expression, bool unique = false) { if (collection.IsNullOrWhiteSpace()) { throw new ArgumentNullException(nameof(collection)); } if (!CollectionIndex.IndexPattern.IsMatch(field)) { throw new ArgumentException("Invalid field format pattern: " + CollectionIndex.IndexPattern.ToString(), "field"); } if (field == "_id") { return(false); // always exists } if (expression != null && expression.Length > 200) { throw new ArgumentException("expression is limited in 200 characters", "expression"); } return(this.Transaction <bool>(collection, true, (col) => { // check if index already exists var current = col.GetIndex(field); // if already exists, just exit if (current != null) { // do not test any difference between current index and new defition return false; } // create index head var index = _indexer.CreateIndex(col); index.Field = field; index.Expression = expression ?? "$." + field; index.Unique = unique; _log.Write(Logger.COMMAND, "create index on '{0}' :: {1} unique: {2}", collection, index.Expression, unique); // read all objects (read from PK index) foreach (var pkNode in new QueryAll("_id", Query.Ascending).Run(col, _indexer)) { // read binary and deserialize document var buffer = _data.Read(pkNode.DataBlock); var doc = _bsonReader.Deserialize(buffer).AsDocument; var expr = new BsonExpression(index.Expression); // get values from expression in document var keys = expr.Execute(doc, true); // adding index node for each value foreach (var key in keys) { // insert new index node var node = _indexer.AddNode(index, key, pkNode); // link index node to datablock node.DataBlock = pkNode.DataBlock; } // check memory usage _trans.CheckPoint(); } return true; })); }
/// <summary> /// Get an IEnumerable of values from a json-like path inside document. Use BsonExpression to parse this path /// </summary> public IEnumerable <BsonValue> Get(string path, bool includeNullIfEmpty = false) { var expr = new BsonExpression(new StringScanner(path), true, true); return(expr.Execute(this, includeNullIfEmpty)); }
/// <summary> /// Implement internal update document /// </summary> private bool UpdateDocument(CollectionPage col, BsonDocument doc) { // normalize id before find var id = doc["_id"]; // validate id for null, min/max values if (id.IsNull || id.IsMinValue || id.IsMaxValue) { throw LiteException.InvalidDataType("_id", id); } _log.Write(Logger.COMMAND, "update document on '{0}' :: _id = {1}", col.CollectionName, id.RawValue); // find indexNode from pk index var pkNode = _indexer.Find(col.PK, id, false, Query.Ascending); // if not found document, no updates if (pkNode == null) { return(false); } // serialize document in bytes var bytes = _bsonWriter.Serialize(doc); // update data storage var dataBlock = _data.Update(col, pkNode.DataBlock, bytes); // get all non-pk index nodes from this data block var allNodes = _indexer.GetNodeList(pkNode, false).ToArray(); // delete/insert indexes - do not touch on PK foreach (var index in col.GetIndexes(false)) { var expr = new BsonExpression(index.Expression); // getting all keys do check var keys = expr.Execute(doc).ToArray(); // get a list of to delete nodes (using ToArray to resolve now) var toDelete = allNodes .Where(x => x.Slot == index.Slot && !keys.Any(k => k == x.Key)) .ToArray(); // get a list of to insert nodes (using ToArray to resolve now) var toInsert = keys .Where(x => !allNodes.Any(k => k.Slot == index.Slot && k.Key == x)) .ToArray(); // delete changed index nodes foreach (var node in toDelete) { _indexer.Delete(index, node.Position); } // insert new nodes foreach (var key in toInsert) { // and add a new one var node = _indexer.AddNode(index, key, pkNode); // link my node to data block node.DataBlock = dataBlock.Position; } } return(true); }
private const int MAX_SORT_PAGES = 5000; // ~ 20Mb? /// <summary> /// EXPERIMENTAL Find with sort operation - use memory or disk (temp file) to sort /// </summary> public List <BsonDocument> FindSort(string collection, Query query, string orderBy, int order = Query.Ascending, int skip = 0, int limit = int.MaxValue) { if (collection.IsNullOrWhiteSpace()) { throw new ArgumentNullException(nameof(collection)); } if (query == null) { throw new ArgumentNullException(nameof(query)); } _log.Write(Logger.COMMAND, "query-sort documents in '{0}' => {1}", collection, query); // evaluate orderBy path/expression var expr = new BsonExpression(orderBy); // lock database for read access using (_locker.Read()) { var last = order == Query.Ascending ? BsonValue.MaxValue : BsonValue.MinValue; var total = limit == int.MaxValue ? int.MaxValue : skip + limit; var indexCounter = 0; var disk = new TempDiskService(); // create memory database using (var engine = new LiteEngine(disk)) { // get collection page var col = this.GetCollectionPage(collection, false); if (col == null) { return(new List <BsonDocument>()); } // create a temp collection in new memory database var tmp = engine._collections.Add("tmp"); // create index pointer var index = engine._indexer.CreateIndex(tmp); // get head/tail index node var head = engine._indexer.GetNode(index.HeadNode); var tail = engine._indexer.GetNode(index.TailNode); // first lets works only with index in query var nodes = query.Run(col, _indexer); foreach (var node in nodes) { var buffer = _data.Read(node.DataBlock); var doc = _bsonReader.Deserialize(buffer).AsDocument; // if needs use filter if (query.UseFilter && query.FilterDocument(doc) == false) { continue; } // get key to be sorted var key = expr.Execute(doc, true).First(); var diff = key.CompareTo(last); // add to list only if lower than last space if ((order == Query.Ascending && diff < 1) || (order == Query.Descending && diff > -1)) { var tmpNode = engine._indexer.AddNode(index, key, null); tmpNode.DataBlock = node.DataBlock; tmpNode.CacheDocument = doc; indexCounter++; // exceeded limit if (indexCounter > total) { var exceeded = (order == Query.Ascending) ? tail.Prev[0] : head.Next[0]; engine._indexer.Delete(index, exceeded); var lnode = (order == Query.Ascending) ? tail.Prev[0] : head.Next[0]; last = engine._indexer.GetNode(lnode).Key; indexCounter--; } // if memory pages excedded limit size, flush to disk if (engine._cache.DirtyUsed > MAX_SORT_PAGES) { engine._trans.PersistDirtyPages(); engine._trans.CheckPoint(); } } } var result = new List <BsonDocument>(); // if skip is lower than limit, take nodes from skip from begin // if skip is higher than limit, take nodes from end and revert order (avoid lots of skip) var find = skip < limit? engine._indexer.FindAll(index, order).Skip(skip).Take(limit) : // get from original order engine._indexer.FindAll(index, -order).Take(limit).Reverse(); // avoid long skips, take from end and revert // --- foreach (var node in engine._indexer.FindAll(index, order).Skip(skip).Take(limit)) foreach (var node in find) { // if document are in cache, use it. if not, get from disk again var doc = node.CacheDocument; if (doc == null) { var buffer = _data.Read(node.DataBlock); doc = _bsonReader.Deserialize(buffer).AsDocument; } result.Add(doc); } return(result); } } }