protected override void ReadContent(ByteReader reader) { var info = reader.ReadString(HEADER_INFO.Length); var ver = reader.ReadByte(); if (info != HEADER_INFO) { throw UltraLiteException.InvalidDatabase(); } if (ver != FILE_VERSION) { throw UltraLiteException.InvalidDatabaseVersion(ver); } this.ChangeID = reader.ReadUInt16(); this.FreeEmptyPageID = reader.ReadUInt32(); this.LastPageID = reader.ReadUInt32(); this.UserVersion = reader.ReadUInt16(); this.Password = reader.ReadBytes(this.Password.Length); this.Salt = reader.ReadBytes(this.Salt.Length); // read page collections references (position on end of page) var cols = reader.ReadByte(); for (var i = 0; i < cols; i++) { this.CollectionPages.Add(reader.ReadString(), reader.ReadUInt32()); } // use last page byte position for recovery mode only because i forgot to reserve area before collection names! // TODO: fix this in next change data structure reader.Position = BasePage.PAGE_SIZE - 1; this.Recovery = reader.ReadBoolean(); }
/// <summary> /// If EOF throw an invalid token exception (used in while()) otherwise return "false" (not EOF) /// </summary> public bool CheckEOF() { if (_eof) { throw UltraLiteException.UnexpectedToken(this.Current); } return(false); }
public Token Expect(string value, bool ignoreCase = true) { if (!this.Is(value, ignoreCase)) { throw UltraLiteException.UnexpectedToken(this, value); } return(this); }
/// <summary> /// Expect for type1 OR type2 OR type3 (if not, throw UnexpectedToken) /// </summary> public Token Expect(TokenType type1, TokenType type2, TokenType type3) { if (this.Type != type1 && this.Type != type2 && this.Type != type3) { throw UltraLiteException.UnexpectedToken(this); } return(this); }
/// <summary> /// Throw syntax exception if not terminate string /// </summary> public void ThrowIfNotFinish() { this.Scan(@"\s*"); if (!this.HasTerminated) { throw UltraLiteException.SyntaxError(this); } }
/// <summary> /// Set datafile length /// </summary> public void SetLength(long fileSize) { // checks if new fileSize will exceed limit size if (fileSize > _options.LimitSize) { throw UltraLiteException.FileSizeExceeded(_options.LimitSize); } // fileSize parameter tell me final size of data file - helpful to extend first datafile _stream.SetLength(fileSize); }
/// <summary> /// Returns first free index slot to be used /// </summary> public CollectionIndex GetFreeIndex() { for (byte i = 0; i < this.Indexes.Length; i++) { if (this.Indexes[i].IsEmpty) { return(this.Indexes[i]); } } throw UltraLiteException.IndexLimitExceeded(this.CollectionName); }
/// <summary> /// Initialize UltraLiteEngine using custom disk service implementation and full engine options /// </summary> public UltraLiteEngine(IDiskService disk, string password = null, TimeSpan?timeout = null, int cacheSize = 5000, Logger log = null, bool utcDate = false) { if (disk == null) { throw new ArgumentNullException(nameof(disk)); } _timeout = timeout ?? TimeSpan.FromMinutes(1); _cacheSize = cacheSize; _disk = disk; _log = log ?? new Logger(); try { // initialize datafile (create) and set log instance _disk.Initialize(_log, password); var buffer = _disk.ReadPage(0); // create header instance from array bytes var header = BasePage.ReadPage(buffer) as HeaderPage; // hash password with sha1 or keep as empty byte[20] var sha1 = password == null ? new byte[20] : AesEncryption.HashSHA1(password); // compare header password with user password even if not passed password (datafile can have password) if (sha1.BinaryCompareTo(header.Password) != 0) { throw UltraLiteException.DatabaseWrongPassword(); } // initialize AES encryptor if (password != null) { _crypto = new AesEncryption(password, header.Salt); } // initialize all services this.InitializeServices(); // if header are marked with recovery, do it now if (header.Recovery) { _trans.Recovery(); } } catch (Exception) { // explicit dispose this.Dispose(); throw; } }
/// <summary> /// Drop an index from a collection /// </summary> public bool DropIndex(string collection, string field) { if (collection.IsNullOrWhiteSpace()) { throw new ArgumentNullException(nameof(collection)); } if (field.IsNullOrWhiteSpace()) { throw new ArgumentNullException(nameof(field)); } if (field == "_id") { throw UltraLiteException.IndexDropId(); } return(this.Transaction <bool>(collection, false, (col) => { // no collection, no index if (col == null) { return false; } // search for index reference var index = col.GetIndex(field); // no index, no drop if (index == null) { return false; } _log.Write(Logger.COMMAND, "drop index on '{0}' :: '{1}'", collection, field); // delete all data pages + indexes pages _indexer.DropIndex(index); // clear index reference index.Clear(); // mark collection page as dirty _pager.SetDirty(col); return true; })); }
/// <summary> /// Try execute some action while has lock exception /// </summary> public static void TryExec(Action action, TimeSpan timeout) { var timer = DateTime.UtcNow.Add(timeout); do { try { action(); return; } catch (IOException ex) { ex.WaitIfLocked(25); } }while (DateTime.UtcNow < timer); throw UltraLiteException.LockTimeout(timeout); }
/// <summary> /// Rename a collection /// </summary> public bool RenameCollection(string collection, string newName) { if (collection.IsNullOrWhiteSpace()) { throw new ArgumentNullException(nameof(collection)); } if (newName.IsNullOrWhiteSpace()) { throw new ArgumentNullException(nameof(newName)); } return(this.Transaction <bool>(collection, false, (col) => { if (col == null) { return false; } _log.Write(Logger.COMMAND, "rename collection '{0}' -> '{1}'", collection, newName); // check if newName already exists if (this.GetCollectionNames().Contains(newName, StringComparer.OrdinalIgnoreCase)) { throw UltraLiteException.AlreadyExistsCollectionName(newName); } // change collection name and save col.CollectionName = newName; // set collection page as dirty _pager.SetDirty(col); // update header collection reference var header = _pager.GetPage <HeaderPage>(0); header.CollectionPages.Remove(collection); header.CollectionPages.Add(newName, col.PageID); _pager.SetDirty(header); return true; })); }
/// <summary> /// Add a new collection. Check if name the not exists /// </summary> public CollectionPage Add(string name) { if (string.IsNullOrEmpty(name)) { throw new ArgumentNullException(nameof(name)); } if (!CollectionPage.NamePattern.IsMatch(name)) { throw UltraLiteException.InvalidFormat(name); } _log.Write(Logger.COMMAND, "creating new collection '{0}'", name); // get header marked as dirty because I will use header after (and NewPage can get another header instance) var header = _pager.GetPage <HeaderPage>(0); // check limit count (8 bytes per collection = 4 to string length, 4 for uint pageID) if (header.CollectionPages.Sum(x => x.Key.Length + 8) + name.Length + 8 >= CollectionPage.MAX_COLLECTIONS_SIZE) { throw UltraLiteException.CollectionLimitExceeded(CollectionPage.MAX_COLLECTIONS_SIZE); } // get new collection page (marked as dirty) var col = _pager.NewPage <CollectionPage>(); // add this page to header page collection header.CollectionPages.Add(name, col.PageID); col.CollectionName = name; // set header page as dirty _pager.SetDirty(header); // create PK index var pk = _indexer.CreateIndex(col); pk.Field = "_id"; pk.Unique = true; return(col); }
/// <summary> /// Read a page with correct instance page object. Checks for pageType /// </summary> public static BasePage ReadPage(byte[] buffer) { var reader = new ByteReader(buffer); var pageID = reader.ReadUInt32(); var pageType = (PageType)reader.ReadByte(); if (pageID == 0 && (byte)pageType > 5) { throw UltraLiteException.InvalidDatabase(); } var page = CreateInstance(pageID, pageType); page.ReadHeader(reader); page.ReadContent(reader); page.DiskData = buffer; return(page); }
internal BsonValue ReadValue(Token token) { switch (token.Type) { case TokenType.String: return(token.Value); case TokenType.OpenBrace: return(this.ReadObject()); case TokenType.OpenBracket: return(this.ReadArray()); case TokenType.Minus: // read next token (must be a number) var number = _tokenizer.ReadToken(false).Expect(TokenType.Int, TokenType.Double); return(number.Type == TokenType.Double ? new BsonValue(-Convert.ToDouble(number.Value, _numberFormat)) : new BsonValue(-Convert.ToInt32(number.Value, _numberFormat))); case TokenType.Int: return(new BsonValue(Convert.ToInt32(token.Value, _numberFormat))); case TokenType.Double: return(new BsonValue(Convert.ToDouble(token.Value, _numberFormat))); case TokenType.Word: switch (token.Value.ToLower()) { case "null": return(BsonValue.Null); case "true": return(true); case "false": return(false); default: throw UltraLiteException.UnexpectedToken(token); } } throw UltraLiteException.UnexpectedToken(token); }
/// <summary> /// Internal implementation of insert a document /// </summary> private void InsertDocument(CollectionPage col, BsonDocument doc, BsonAutoId autoId) { // collection Sequence was created after release current datafile version. // In this case, Sequence will be 0 but already has documents. Let's fix this // ** this code can be removed when datafile change from 7 (HeaderPage.FILE_VERSION) ** if (col.Sequence == 0 && col.DocumentCount > 0) { var max = this.Max(col.CollectionName, "_id"); // if max value is a number, convert to Sequence last value // if not, just set sequence as document count col.Sequence = (max.IsInt32 || max.IsInt64 || max.IsDouble || max.IsDecimal) ? Convert.ToInt64(max.RawValue) : Convert.ToInt64(col.DocumentCount); } // increase collection sequence _id col.Sequence++; _pager.SetDirty(col); // if no _id, add one if (!doc.RawValue.TryGetValue("_id", out var id)) { doc["_id"] = id = autoId == BsonAutoId.ObjectId ? new BsonValue(ObjectId.NewObjectId()) : autoId == BsonAutoId.Guid ? new BsonValue(Guid.NewGuid()) : autoId == BsonAutoId.Int32 ? new BsonValue((Int32)col.Sequence) : autoId == BsonAutoId.Int64 ? new BsonValue(col.Sequence) : BsonValue.Null; } // create bubble in sequence number if _id is bigger than current sequence else if (autoId == BsonAutoId.Int32 || autoId == BsonAutoId.Int64) { var current = id.AsInt64; // if current id is bigger than sequence, jump sequence to this number. Other was, do not increse sequnce col.Sequence = current >= col.Sequence ? current : col.Sequence - 1; } // test if _id is a valid type if (id.IsNull || id.IsMinValue || id.IsMaxValue) { throw UltraLiteException.InvalidDataType("_id", id); } _log.Write(Logger.COMMAND, "insert document on '{0}' :: _id = {1}", col.CollectionName, id.RawValue); // serialize object var bytes = BsonWriter.Serialize(doc); // storage in data pages - returns dataBlock address var dataBlock = _data.Insert(col, bytes); // store id in a PK index [0 array] var pk = _indexer.AddNode(col.PK, id, null); // do link between index <-> data block pk.DataBlock = dataBlock.Position; // for each index, insert new IndexNode foreach (var index in col.GetIndexes(false)) { // for each index, get all keys (support now multi-key) - gets distinct values only // if index are unique, get single key only var expr = new BsonFields(index.Field); var keys = expr.Execute(doc, true); // do a loop with all keys (multi-key supported) foreach (var key in keys) { // insert node var node = _indexer.AddNode(index, key, pk); // link my index node to data block address node.DataBlock = dataBlock.Position; } } }
/// <summary> /// Insert a new node index inside an collection index. /// </summary> private IndexNode AddNode(CollectionIndex index, BsonValue key, byte level, IndexNode last) { // calc key size var keyLength = key.GetBytesCount(false); // test for index key maxlength if (keyLength > MAX_INDEX_LENGTH) { throw UltraLiteException.IndexKeyTooLong(); } // creating a new index node var node = new IndexNode(level) { Key = key, KeyLength = (ushort)keyLength, Slot = (byte)index.Slot }; // get a free page to insert my index node var page = _pager.GetFreePage <IndexPage>(index.FreeIndexPageID, node.Length); node.Page = page; // add index node to page page.AddNode(node); // now, let's link my index node on right place var cur = this.GetNode(index.HeadNode); // using as cache last IndexNode cache = null; // scan from top left for (var i = index.MaxLevel - 1; i >= 0; i--) { // get cache for last node cache = cache != null && cache.Position.Equals(cur.Next[i]) ? cache : this.GetNode(cur.Next[i]); // for(; <while_not_this>; <do_this>) { ... } for (; cur.Next[i].IsEmpty == false; cur = cache) { // get cache for last node cache = cache != null && cache.Position.Equals(cur.Next[i]) ? cache : this.GetNode(cur.Next[i]); // read next node to compare var diff = cache.Key.CompareTo(key); // if unique and diff = 0, throw index exception (must rollback transaction - others nodes can be dirty) if (diff == 0 && index.Unique) { throw UltraLiteException.IndexDuplicateKey(index.Field, key); } if (diff == 1) { break; } } if (i <= (level - 1)) // level == length { // cur = current (immediately before - prev) // node = new inserted node // next = next node (where cur is pointing) _pager.SetDirty(cur.Page); node.Next[i] = cur.Next[i]; node.Prev[i] = cur.Position; cur.Next[i] = node.Position; var next = this.GetNode(node.Next[i]); if (next != null) { next.Prev[i] = node.Position; _pager.SetDirty(next.Page); } } } // add/remove indexPage on freelist if has space _pager.AddOrRemoveToFreeList(page.FreeBytes > IndexPage.INDEX_RESERVED_BYTES, page, index.Page, ref index.FreeIndexPageID); // if last node exists, create a double link list if (last != null) { // link new node with last node if (last.NextNode.IsEmpty == false) { // fix link pointer with has more nodes in list var next = this.GetNode(last.NextNode); next.PrevNode = node.Position; last.NextNode = node.Position; node.PrevNode = last.Position; node.NextNode = next.Position; _pager.SetDirty(next.Page); } else { last.NextNode = node.Position; node.PrevNode = last.Position; } // set last node page as dirty _pager.SetDirty(last.Page); } return(node); }
/// <summary> /// Create a new instance from a Type /// </summary> public static object CreateInstance(Type type) { try { if (_cacheCtor.TryGetValue(type, out CreateObject c)) { return(c()); } } catch (Exception ex) { throw UltraLiteException.InvalidCtor(type, ex); } lock (_cacheCtor) { try { if (_cacheCtor.TryGetValue(type, out CreateObject c)) { return(c()); } if (type.GetTypeInfo().IsClass) { _cacheCtor.Add(type, c = CreateClass(type)); } else if (type.GetTypeInfo().IsInterface) // some know interfaces { if (type.GetTypeInfo().IsGenericType) { var typeDef = type.GetGenericTypeDefinition(); if (typeDef == typeof(IList <>) || typeDef == typeof(ICollection <>) || typeDef == typeof(IEnumerable <>)) { return(CreateInstance(GetGenericListOfType(UnderlyingTypeOf(type)))); } else if (typeDef == typeof(IDictionary <,>)) { var k = type.GetTypeInfo().GetGenericArguments()[0]; var v = type.GetTypeInfo().GetGenericArguments()[1]; return(CreateInstance(GetGenericDictionaryOfType(k, v))); } } throw UltraLiteException.InvalidCtor(type, null); } else // structs { _cacheCtor.Add(type, c = CreateStruct(type)); } return(c()); } catch (Exception ex) { throw UltraLiteException.InvalidCtor(type, ex); } } }
internal BsonValue Serialize(Type type, object obj, int depth) { if (++depth > MAX_DEPTH) { throw UltraLiteException.DocumentMaxDepth(MAX_DEPTH, type); } if (obj == null) { return(BsonValue.Null); } Func <object, BsonValue> custom; // if is already a bson value if (obj is BsonValue) { return(new BsonValue((BsonValue)obj)); } // test string - mapper has some special options else if (obj is String) { var str = this.TrimWhitespace ? (obj as String).Trim() : (String)obj; if (this.EmptyStringToNull && str.Length == 0) { return(BsonValue.Null); } else { return(new BsonValue(str)); } } // basic Bson data types (cast datatype for better performance optimization) else if (obj is Int32) { return(new BsonValue((Int32)obj)); } else if (obj is Int64) { return(new BsonValue((Int64)obj)); } else if (obj is Double) { return(new BsonValue((Double)obj)); } else if (obj is Decimal) { return(new BsonValue((Decimal)obj)); } else if (obj is Byte[]) { return(new BsonValue((Byte[])obj)); } else if (obj is ObjectId) { return(new BsonValue((ObjectId)obj)); } else if (obj is Guid) { return(new BsonValue((Guid)obj)); } else if (obj is Boolean) { return(new BsonValue((Boolean)obj)); } else if (obj is DateTime) { return(new BsonValue((DateTime)obj)); } // basic .net type to convert to bson else if (obj is Int16 || obj is UInt16 || obj is Byte || obj is SByte) { return(new BsonValue(Convert.ToInt32(obj))); } else if (obj is UInt32) { return(new BsonValue(Convert.ToInt64(obj))); } else if (obj is UInt64) { var ulng = ((UInt64)obj); var lng = unchecked ((Int64)ulng); return(new BsonValue(lng)); } else if (obj is Single) { return(new BsonValue(Convert.ToDouble(obj))); } else if (obj is Char || obj is Enum) { return(new BsonValue(obj.ToString())); } // check if is a custom type else if (_customSerializer.TryGetValue(type, out custom) || _customSerializer.TryGetValue(obj.GetType(), out custom)) { return(custom(obj)); } // for dictionary else if (obj is IDictionary) { // when you are converting Dictionary<string, object> if (type == typeof(object)) { type = obj.GetType(); } var itemType = type.GetTypeInfo().GetGenericArguments()[1]; return(this.SerializeDictionary(itemType, obj as IDictionary, depth)); } // check if is a list or array else if (obj is IEnumerable) { return(this.SerializeArray(Reflection.GetListItemType(obj.GetType()), obj as IEnumerable, depth)); } // otherwise serialize as a plain object else { return(this.SerializeObject(type, obj, depth)); } }
internal object Deserialize(Type type, BsonValue value) { Func <BsonValue, object> custom; // null value - null returns if (value.IsNull) { return(null); } // if is nullable, get underlying type else if (Reflection.IsNullable(type)) { type = Reflection.UnderlyingTypeOf(type); } // check if your type is already a BsonValue/BsonDocument/BsonArray if (type == typeof(BsonValue)) { return(new BsonValue(value)); } else if (type == typeof(BsonDocument)) { return(value.AsDocument); } else if (type == typeof(BsonArray)) { return(value.AsArray); } // raw values to native bson values else if (_bsonTypes.Contains(type)) { return(value.RawValue); } // simple ConvertTo to basic .NET types else if (_basicTypes.Contains(type)) { return(Convert.ChangeType(value.RawValue, type)); } // special cast to UInt64 to Int64 else if (type == typeof(UInt64)) { return(unchecked ((UInt64)((Int64)value.RawValue))); } // enum value is an int else if (type.GetTypeInfo().IsEnum) { return(Enum.Parse(type, value.AsString)); } // test if has a custom type implementation else if (_customDeserializer.TryGetValue(type, out custom)) { return(custom(value)); } // if value is array, deserialize as array else if (value.IsArray) { // when array are from an object (like in Dictionary<string, object> { ["array"] = new string[] { "a", "b" } if (type == typeof(object)) { return(this.DeserializeArray(typeof(object), value.AsArray)); } if (type.IsArray) { return(this.DeserializeArray(type.GetElementType(), value.AsArray)); } else { return(this.DeserializeList(type, value.AsArray)); } } // if value is document, deserialize as document else if (value.IsDocument) { BsonValue typeField; var doc = value.AsDocument; // test if value is object and has _type if (doc.RawValue.TryGetValue("_type", out typeField)) { type = Type.GetType(typeField.AsString); if (type == null) { throw UltraLiteException.InvalidTypedName(typeField.AsString); } } // when complex type has no definition (== typeof(object)) use Dictionary<string, object> to better set values else if (type == typeof(object)) { type = typeof(Dictionary <string, object>); } var o = _typeInstantiator(type); if (o is IDictionary && type.GetTypeInfo().IsGenericType) { var k = type.GetTypeInfo().GetGenericArguments()[0]; var t = type.GetTypeInfo().GetGenericArguments()[1]; this.DeserializeDictionary(k, t, (IDictionary)o, value.AsDocument); } else { this.DeserializeObject(type, o, doc); } return(o); } // in last case, return value as-is - can cause "cast error" // it's used for "public object MyInt { get; set; }" return(value.RawValue); }
/// <summary> /// Implement internal update document /// </summary> private bool UpdateDocument(CollectionPage col, BsonDocument doc) { // normalize id before find var id = doc["_id"]; // validate id for null, min/max values if (id.IsNull || id.IsMinValue || id.IsMaxValue) { throw UltraLiteException.InvalidDataType("_id", id); } _log.Write(Logger.COMMAND, "update document on '{0}' :: _id = {1}", col.CollectionName, id.RawValue); // find indexNode from pk index var pkNode = _indexer.Find(col.PK, id, false, Query.Ascending); // if not found document, no updates if (pkNode == null) { return(false); } // serialize document in bytes var bytes = BsonWriter.Serialize(doc); // update data storage var dataBlock = _data.Update(col, pkNode.DataBlock, bytes); // get all non-pk index nodes from this data block var allNodes = _indexer.GetNodeList(pkNode, false).ToArray(); // delete/insert indexes - do not touch on PK foreach (var index in col.GetIndexes(false)) { var expr = new BsonFields(index.Field); // getting all keys do check var keys = expr.Execute(doc).ToArray(); // get a list of to delete nodes (using ToArray to resolve now) var toDelete = allNodes .Where(x => x.Slot == index.Slot && !keys.Any(k => k == x.Key)) .ToArray(); // get a list of to insert nodes (using ToArray to resolve now) var toInsert = keys .Where(x => !allNodes.Any(k => k.Slot == index.Slot && k.Key == x)) .ToArray(); // delete changed index nodes foreach (var node in toDelete) { _indexer.Delete(index, node.Position); } // insert new nodes foreach (var key in toInsert) { // and add a new one var node = _indexer.AddNode(index, key, pkNode); // link my node to data block node.DataBlock = dataBlock.Position; } } return(true); }