protected override void ReadContent(ByteReader reader)
{
var info = reader.ReadString(HEADER_INFO.Length);
var ver = reader.ReadByte();
if (info != HEADER_INFO)
{
throw UltraLiteException.InvalidDatabase();
}
if (ver != FILE_VERSION)
{
throw UltraLiteException.InvalidDatabaseVersion(ver);
}
this.ChangeID = reader.ReadUInt16();
this.FreeEmptyPageID = reader.ReadUInt32();
this.LastPageID = reader.ReadUInt32();
this.UserVersion = reader.ReadUInt16();
this.Password = reader.ReadBytes(this.Password.Length);
this.Salt = reader.ReadBytes(this.Salt.Length);
// read page collections references (position on end of page)
var cols = reader.ReadByte();
for (var i = 0; i < cols; i++)
{
this.CollectionPages.Add(reader.ReadString(), reader.ReadUInt32());
}
// use last page byte position for recovery mode only because i forgot to reserve area before collection names!
// TODO: fix this in next change data structure
reader.Position = BasePage.PAGE_SIZE - 1;
this.Recovery = reader.ReadBoolean();
}
/// <summary>
/// If EOF throw an invalid token exception (used in while()) otherwise return "false" (not EOF)
/// </summary>
public bool CheckEOF()
{
if (_eof)
{
throw UltraLiteException.UnexpectedToken(this.Current);
}
return(false);
}
public Token Expect(string value, bool ignoreCase = true)
{
if (!this.Is(value, ignoreCase))
{
throw UltraLiteException.UnexpectedToken(this, value);
}
return(this);
}
/// <summary>
/// Expect for type1 OR type2 OR type3 (if not, throw UnexpectedToken)
/// </summary>
public Token Expect(TokenType type1, TokenType type2, TokenType type3)
{
if (this.Type != type1 && this.Type != type2 && this.Type != type3)
{
throw UltraLiteException.UnexpectedToken(this);
}
return(this);
}
/// <summary>
/// Throw syntax exception if not terminate string
/// </summary>
public void ThrowIfNotFinish()
{
this.Scan(@"\s*");
if (!this.HasTerminated)
{
throw UltraLiteException.SyntaxError(this);
}
}
/// <summary>
/// Set datafile length
/// </summary>
public void SetLength(long fileSize)
{
// checks if new fileSize will exceed limit size
if (fileSize > _options.LimitSize)
{
throw UltraLiteException.FileSizeExceeded(_options.LimitSize);
}
// fileSize parameter tell me final size of data file - helpful to extend first datafile
_stream.SetLength(fileSize);
}
/// <summary>
/// Returns first free index slot to be used
/// </summary>
public CollectionIndex GetFreeIndex()
{
for (byte i = 0; i < this.Indexes.Length; i++)
{
if (this.Indexes[i].IsEmpty)
{
return(this.Indexes[i]);
}
}
throw UltraLiteException.IndexLimitExceeded(this.CollectionName);
}
/// <summary>
/// Initialize UltraLiteEngine using custom disk service implementation and full engine options
/// </summary>
public UltraLiteEngine(IDiskService disk, string password = null, TimeSpan?timeout = null, int cacheSize = 5000, Logger log = null, bool utcDate = false)
{
if (disk == null)
{
throw new ArgumentNullException(nameof(disk));
}
_timeout = timeout ?? TimeSpan.FromMinutes(1);
_cacheSize = cacheSize;
_disk = disk;
_log = log ?? new Logger();
try
{
// initialize datafile (create) and set log instance
_disk.Initialize(_log, password);
var buffer = _disk.ReadPage(0);
// create header instance from array bytes
var header = BasePage.ReadPage(buffer) as HeaderPage;
// hash password with sha1 or keep as empty byte[20]
var sha1 = password == null ? new byte[20] : AesEncryption.HashSHA1(password);
// compare header password with user password even if not passed password (datafile can have password)
if (sha1.BinaryCompareTo(header.Password) != 0)
{
throw UltraLiteException.DatabaseWrongPassword();
}
// initialize AES encryptor
if (password != null)
{
_crypto = new AesEncryption(password, header.Salt);
}
// initialize all services
this.InitializeServices();
// if header are marked with recovery, do it now
if (header.Recovery)
{
_trans.Recovery();
}
}
catch (Exception)
{
// explicit dispose
this.Dispose();
throw;
}
}
/// <summary>
/// Drop an index from a collection
/// </summary>
public bool DropIndex(string collection, string field)
{
if (collection.IsNullOrWhiteSpace())
{
throw new ArgumentNullException(nameof(collection));
}
if (field.IsNullOrWhiteSpace())
{
throw new ArgumentNullException(nameof(field));
}
if (field == "_id")
{
throw UltraLiteException.IndexDropId();
}
return(this.Transaction <bool>(collection, false, (col) =>
{
// no collection, no index
if (col == null)
{
return false;
}
// search for index reference
var index = col.GetIndex(field);
// no index, no drop
if (index == null)
{
return false;
}
_log.Write(Logger.COMMAND, "drop index on '{0}' :: '{1}'", collection, field);
// delete all data pages + indexes pages
_indexer.DropIndex(index);
// clear index reference
index.Clear();
// mark collection page as dirty
_pager.SetDirty(col);
return true;
}));
}
/// <summary>
/// Try execute some action while has lock exception
/// </summary>
public static void TryExec(Action action, TimeSpan timeout)
{
var timer = DateTime.UtcNow.Add(timeout);
do
{
try
{
action();
return;
}
catch (IOException ex)
{
ex.WaitIfLocked(25);
}
}while (DateTime.UtcNow < timer);
throw UltraLiteException.LockTimeout(timeout);
}
/// <summary>
/// Rename a collection
/// </summary>
public bool RenameCollection(string collection, string newName)
{
if (collection.IsNullOrWhiteSpace())
{
throw new ArgumentNullException(nameof(collection));
}
if (newName.IsNullOrWhiteSpace())
{
throw new ArgumentNullException(nameof(newName));
}
return(this.Transaction <bool>(collection, false, (col) =>
{
if (col == null)
{
return false;
}
_log.Write(Logger.COMMAND, "rename collection '{0}' -> '{1}'", collection, newName);
// check if newName already exists
if (this.GetCollectionNames().Contains(newName, StringComparer.OrdinalIgnoreCase))
{
throw UltraLiteException.AlreadyExistsCollectionName(newName);
}
// change collection name and save
col.CollectionName = newName;
// set collection page as dirty
_pager.SetDirty(col);
// update header collection reference
var header = _pager.GetPage <HeaderPage>(0);
header.CollectionPages.Remove(collection);
header.CollectionPages.Add(newName, col.PageID);
_pager.SetDirty(header);
return true;
}));
}
/// <summary>
/// Add a new collection. Check if name the not exists
/// </summary>
public CollectionPage Add(string name)
{
if (string.IsNullOrEmpty(name))
{
throw new ArgumentNullException(nameof(name));
}
if (!CollectionPage.NamePattern.IsMatch(name))
{
throw UltraLiteException.InvalidFormat(name);
}
_log.Write(Logger.COMMAND, "creating new collection '{0}'", name);
// get header marked as dirty because I will use header after (and NewPage can get another header instance)
var header = _pager.GetPage <HeaderPage>(0);
// check limit count (8 bytes per collection = 4 to string length, 4 for uint pageID)
if (header.CollectionPages.Sum(x => x.Key.Length + 8) + name.Length + 8 >= CollectionPage.MAX_COLLECTIONS_SIZE)
{
throw UltraLiteException.CollectionLimitExceeded(CollectionPage.MAX_COLLECTIONS_SIZE);
}
// get new collection page (marked as dirty)
var col = _pager.NewPage <CollectionPage>();
// add this page to header page collection
header.CollectionPages.Add(name, col.PageID);
col.CollectionName = name;
// set header page as dirty
_pager.SetDirty(header);
// create PK index
var pk = _indexer.CreateIndex(col);
pk.Field = "_id";
pk.Unique = true;
return(col);
}
/// <summary>
/// Read a page with correct instance page object. Checks for pageType
/// </summary>
public static BasePage ReadPage(byte[] buffer)
{
var reader = new ByteReader(buffer);
var pageID = reader.ReadUInt32();
var pageType = (PageType)reader.ReadByte();
if (pageID == 0 && (byte)pageType > 5)
{
throw UltraLiteException.InvalidDatabase();
}
var page = CreateInstance(pageID, pageType);
page.ReadHeader(reader);
page.ReadContent(reader);
page.DiskData = buffer;
return(page);
}
internal BsonValue ReadValue(Token token)
{
switch (token.Type)
{
case TokenType.String: return(token.Value);
case TokenType.OpenBrace: return(this.ReadObject());
case TokenType.OpenBracket: return(this.ReadArray());
case TokenType.Minus:
// read next token (must be a number)
var number = _tokenizer.ReadToken(false).Expect(TokenType.Int, TokenType.Double);
return(number.Type == TokenType.Double ?
new BsonValue(-Convert.ToDouble(number.Value, _numberFormat)) :
new BsonValue(-Convert.ToInt32(number.Value, _numberFormat)));
case TokenType.Int: return(new BsonValue(Convert.ToInt32(token.Value, _numberFormat)));
case TokenType.Double: return(new BsonValue(Convert.ToDouble(token.Value, _numberFormat)));
case TokenType.Word:
switch (token.Value.ToLower())
{
case "null": return(BsonValue.Null);
case "true": return(true);
case "false": return(false);
default: throw UltraLiteException.UnexpectedToken(token);
}
}
throw UltraLiteException.UnexpectedToken(token);
}
/// <summary>
/// Internal implementation of insert a document
/// </summary>
private void InsertDocument(CollectionPage col, BsonDocument doc, BsonAutoId autoId)
{
// collection Sequence was created after release current datafile version.
// In this case, Sequence will be 0 but already has documents. Let's fix this
// ** this code can be removed when datafile change from 7 (HeaderPage.FILE_VERSION) **
if (col.Sequence == 0 && col.DocumentCount > 0)
{
var max = this.Max(col.CollectionName, "_id");
// if max value is a number, convert to Sequence last value
// if not, just set sequence as document count
col.Sequence = (max.IsInt32 || max.IsInt64 || max.IsDouble || max.IsDecimal) ?
Convert.ToInt64(max.RawValue) :
Convert.ToInt64(col.DocumentCount);
}
// increase collection sequence _id
col.Sequence++;
_pager.SetDirty(col);
// if no _id, add one
if (!doc.RawValue.TryGetValue("_id", out var id))
{
doc["_id"] = id =
autoId == BsonAutoId.ObjectId ? new BsonValue(ObjectId.NewObjectId()) :
autoId == BsonAutoId.Guid ? new BsonValue(Guid.NewGuid()) :
autoId == BsonAutoId.Int32 ? new BsonValue((Int32)col.Sequence) :
autoId == BsonAutoId.Int64 ? new BsonValue(col.Sequence) : BsonValue.Null;
}
// create bubble in sequence number if _id is bigger than current sequence
else if (autoId == BsonAutoId.Int32 || autoId == BsonAutoId.Int64)
{
var current = id.AsInt64;
// if current id is bigger than sequence, jump sequence to this number. Other was, do not increse sequnce
col.Sequence = current >= col.Sequence ? current : col.Sequence - 1;
}
// test if _id is a valid type
if (id.IsNull || id.IsMinValue || id.IsMaxValue)
{
throw UltraLiteException.InvalidDataType("_id", id);
}
_log.Write(Logger.COMMAND, "insert document on '{0}' :: _id = {1}", col.CollectionName, id.RawValue);
// serialize object
var bytes = BsonWriter.Serialize(doc);
// storage in data pages - returns dataBlock address
var dataBlock = _data.Insert(col, bytes);
// store id in a PK index [0 array]
var pk = _indexer.AddNode(col.PK, id, null);
// do link between index <-> data block
pk.DataBlock = dataBlock.Position;
// for each index, insert new IndexNode
foreach (var index in col.GetIndexes(false))
{
// for each index, get all keys (support now multi-key) - gets distinct values only
// if index are unique, get single key only
var expr = new BsonFields(index.Field);
var keys = expr.Execute(doc, true);
// do a loop with all keys (multi-key supported)
foreach (var key in keys)
{
// insert node
var node = _indexer.AddNode(index, key, pk);
// link my index node to data block address
node.DataBlock = dataBlock.Position;
}
}
}
/// <summary>
/// Insert a new node index inside an collection index.
/// </summary>
private IndexNode AddNode(CollectionIndex index, BsonValue key, byte level, IndexNode last)
{
// calc key size
var keyLength = key.GetBytesCount(false);
// test for index key maxlength
if (keyLength > MAX_INDEX_LENGTH)
{
throw UltraLiteException.IndexKeyTooLong();
}
// creating a new index node
var node = new IndexNode(level)
{
Key = key,
KeyLength = (ushort)keyLength,
Slot = (byte)index.Slot
};
// get a free page to insert my index node
var page = _pager.GetFreePage <IndexPage>(index.FreeIndexPageID, node.Length);
node.Page = page;
// add index node to page
page.AddNode(node);
// now, let's link my index node on right place
var cur = this.GetNode(index.HeadNode);
// using as cache last
IndexNode cache = null;
// scan from top left
for (var i = index.MaxLevel - 1; i >= 0; i--)
{
// get cache for last node
cache = cache != null && cache.Position.Equals(cur.Next[i]) ? cache : this.GetNode(cur.Next[i]);
// for(; <while_not_this>; <do_this>) { ... }
for (; cur.Next[i].IsEmpty == false; cur = cache)
{
// get cache for last node
cache = cache != null && cache.Position.Equals(cur.Next[i]) ? cache : this.GetNode(cur.Next[i]);
// read next node to compare
var diff = cache.Key.CompareTo(key);
// if unique and diff = 0, throw index exception (must rollback transaction - others nodes can be dirty)
if (diff == 0 && index.Unique)
{
throw UltraLiteException.IndexDuplicateKey(index.Field, key);
}
if (diff == 1)
{
break;
}
}
if (i <= (level - 1)) // level == length
{
// cur = current (immediately before - prev)
// node = new inserted node
// next = next node (where cur is pointing)
_pager.SetDirty(cur.Page);
node.Next[i] = cur.Next[i];
node.Prev[i] = cur.Position;
cur.Next[i] = node.Position;
var next = this.GetNode(node.Next[i]);
if (next != null)
{
next.Prev[i] = node.Position;
_pager.SetDirty(next.Page);
}
}
}
// add/remove indexPage on freelist if has space
_pager.AddOrRemoveToFreeList(page.FreeBytes > IndexPage.INDEX_RESERVED_BYTES, page, index.Page, ref index.FreeIndexPageID);
// if last node exists, create a double link list
if (last != null)
{
// link new node with last node
if (last.NextNode.IsEmpty == false)
{
// fix link pointer with has more nodes in list
var next = this.GetNode(last.NextNode);
next.PrevNode = node.Position;
last.NextNode = node.Position;
node.PrevNode = last.Position;
node.NextNode = next.Position;
_pager.SetDirty(next.Page);
}
else
{
last.NextNode = node.Position;
node.PrevNode = last.Position;
}
// set last node page as dirty
_pager.SetDirty(last.Page);
}
return(node);
}
/// <summary>
/// Create a new instance from a Type
/// </summary>
public static object CreateInstance(Type type)
{
try
{
if (_cacheCtor.TryGetValue(type, out CreateObject c))
{
return(c());
}
}
catch (Exception ex)
{
throw UltraLiteException.InvalidCtor(type, ex);
}
lock (_cacheCtor)
{
try
{
if (_cacheCtor.TryGetValue(type, out CreateObject c))
{
return(c());
}
if (type.GetTypeInfo().IsClass)
{
_cacheCtor.Add(type, c = CreateClass(type));
}
else if (type.GetTypeInfo().IsInterface) // some know interfaces
{
if (type.GetTypeInfo().IsGenericType)
{
var typeDef = type.GetGenericTypeDefinition();
if (typeDef == typeof(IList <>) ||
typeDef == typeof(ICollection <>) ||
typeDef == typeof(IEnumerable <>))
{
return(CreateInstance(GetGenericListOfType(UnderlyingTypeOf(type))));
}
else if (typeDef == typeof(IDictionary <,>))
{
var k = type.GetTypeInfo().GetGenericArguments()[0];
var v = type.GetTypeInfo().GetGenericArguments()[1];
return(CreateInstance(GetGenericDictionaryOfType(k, v)));
}
}
throw UltraLiteException.InvalidCtor(type, null);
}
else // structs
{
_cacheCtor.Add(type, c = CreateStruct(type));
}
return(c());
}
catch (Exception ex)
{
throw UltraLiteException.InvalidCtor(type, ex);
}
}
}
internal BsonValue Serialize(Type type, object obj, int depth)
{
if (++depth > MAX_DEPTH)
{
throw UltraLiteException.DocumentMaxDepth(MAX_DEPTH, type);
}
if (obj == null)
{
return(BsonValue.Null);
}
Func <object, BsonValue> custom;
// if is already a bson value
if (obj is BsonValue)
{
return(new BsonValue((BsonValue)obj));
}
// test string - mapper has some special options
else if (obj is String)
{
var str = this.TrimWhitespace ? (obj as String).Trim() : (String)obj;
if (this.EmptyStringToNull && str.Length == 0)
{
return(BsonValue.Null);
}
else
{
return(new BsonValue(str));
}
}
// basic Bson data types (cast datatype for better performance optimization)
else if (obj is Int32)
{
return(new BsonValue((Int32)obj));
}
else if (obj is Int64)
{
return(new BsonValue((Int64)obj));
}
else if (obj is Double)
{
return(new BsonValue((Double)obj));
}
else if (obj is Decimal)
{
return(new BsonValue((Decimal)obj));
}
else if (obj is Byte[])
{
return(new BsonValue((Byte[])obj));
}
else if (obj is ObjectId)
{
return(new BsonValue((ObjectId)obj));
}
else if (obj is Guid)
{
return(new BsonValue((Guid)obj));
}
else if (obj is Boolean)
{
return(new BsonValue((Boolean)obj));
}
else if (obj is DateTime)
{
return(new BsonValue((DateTime)obj));
}
// basic .net type to convert to bson
else if (obj is Int16 || obj is UInt16 || obj is Byte || obj is SByte)
{
return(new BsonValue(Convert.ToInt32(obj)));
}
else if (obj is UInt32)
{
return(new BsonValue(Convert.ToInt64(obj)));
}
else if (obj is UInt64)
{
var ulng = ((UInt64)obj);
var lng = unchecked ((Int64)ulng);
return(new BsonValue(lng));
}
else if (obj is Single)
{
return(new BsonValue(Convert.ToDouble(obj)));
}
else if (obj is Char || obj is Enum)
{
return(new BsonValue(obj.ToString()));
}
// check if is a custom type
else if (_customSerializer.TryGetValue(type, out custom) || _customSerializer.TryGetValue(obj.GetType(), out custom))
{
return(custom(obj));
}
// for dictionary
else if (obj is IDictionary)
{
// when you are converting Dictionary<string, object>
if (type == typeof(object))
{
type = obj.GetType();
}
var itemType = type.GetTypeInfo().GetGenericArguments()[1];
return(this.SerializeDictionary(itemType, obj as IDictionary, depth));
}
// check if is a list or array
else if (obj is IEnumerable)
{
return(this.SerializeArray(Reflection.GetListItemType(obj.GetType()), obj as IEnumerable, depth));
}
// otherwise serialize as a plain object
else
{
return(this.SerializeObject(type, obj, depth));
}
}
internal object Deserialize(Type type, BsonValue value)
{
Func <BsonValue, object> custom;
// null value - null returns
if (value.IsNull)
{
return(null);
}
// if is nullable, get underlying type
else if (Reflection.IsNullable(type))
{
type = Reflection.UnderlyingTypeOf(type);
}
// check if your type is already a BsonValue/BsonDocument/BsonArray
if (type == typeof(BsonValue))
{
return(new BsonValue(value));
}
else if (type == typeof(BsonDocument))
{
return(value.AsDocument);
}
else if (type == typeof(BsonArray))
{
return(value.AsArray);
}
// raw values to native bson values
else if (_bsonTypes.Contains(type))
{
return(value.RawValue);
}
// simple ConvertTo to basic .NET types
else if (_basicTypes.Contains(type))
{
return(Convert.ChangeType(value.RawValue, type));
}
// special cast to UInt64 to Int64
else if (type == typeof(UInt64))
{
return(unchecked ((UInt64)((Int64)value.RawValue)));
}
// enum value is an int
else if (type.GetTypeInfo().IsEnum)
{
return(Enum.Parse(type, value.AsString));
}
// test if has a custom type implementation
else if (_customDeserializer.TryGetValue(type, out custom))
{
return(custom(value));
}
// if value is array, deserialize as array
else if (value.IsArray)
{
// when array are from an object (like in Dictionary<string, object> { ["array"] = new string[] { "a", "b" }
if (type == typeof(object))
{
return(this.DeserializeArray(typeof(object), value.AsArray));
}
if (type.IsArray)
{
return(this.DeserializeArray(type.GetElementType(), value.AsArray));
}
else
{
return(this.DeserializeList(type, value.AsArray));
}
}
// if value is document, deserialize as document
else if (value.IsDocument)
{
BsonValue typeField;
var doc = value.AsDocument;
// test if value is object and has _type
if (doc.RawValue.TryGetValue("_type", out typeField))
{
type = Type.GetType(typeField.AsString);
if (type == null)
{
throw UltraLiteException.InvalidTypedName(typeField.AsString);
}
}
// when complex type has no definition (== typeof(object)) use Dictionary<string, object> to better set values
else if (type == typeof(object))
{
type = typeof(Dictionary <string, object>);
}
var o = _typeInstantiator(type);
if (o is IDictionary && type.GetTypeInfo().IsGenericType)
{
var k = type.GetTypeInfo().GetGenericArguments()[0];
var t = type.GetTypeInfo().GetGenericArguments()[1];
this.DeserializeDictionary(k, t, (IDictionary)o, value.AsDocument);
}
else
{
this.DeserializeObject(type, o, doc);
}
return(o);
}
// in last case, return value as-is - can cause "cast error"
// it's used for "public object MyInt { get; set; }"
return(value.RawValue);
}
/// <summary>
/// Implement internal update document
/// </summary>
private bool UpdateDocument(CollectionPage col, BsonDocument doc)
{
// normalize id before find
var id = doc["_id"];
// validate id for null, min/max values
if (id.IsNull || id.IsMinValue || id.IsMaxValue)
{
throw UltraLiteException.InvalidDataType("_id", id);
}
_log.Write(Logger.COMMAND, "update document on '{0}' :: _id = {1}", col.CollectionName, id.RawValue);
// find indexNode from pk index
var pkNode = _indexer.Find(col.PK, id, false, Query.Ascending);
// if not found document, no updates
if (pkNode == null)
{
return(false);
}
// serialize document in bytes
var bytes = BsonWriter.Serialize(doc);
// update data storage
var dataBlock = _data.Update(col, pkNode.DataBlock, bytes);
// get all non-pk index nodes from this data block
var allNodes = _indexer.GetNodeList(pkNode, false).ToArray();
// delete/insert indexes - do not touch on PK
foreach (var index in col.GetIndexes(false))
{
var expr = new BsonFields(index.Field);
// getting all keys do check
var keys = expr.Execute(doc).ToArray();
// get a list of to delete nodes (using ToArray to resolve now)
var toDelete = allNodes
.Where(x => x.Slot == index.Slot && !keys.Any(k => k == x.Key))
.ToArray();
// get a list of to insert nodes (using ToArray to resolve now)
var toInsert = keys
.Where(x => !allNodes.Any(k => k.Slot == index.Slot && k.Key == x))
.ToArray();
// delete changed index nodes
foreach (var node in toDelete)
{
_indexer.Delete(index, node.Position);
}
// insert new nodes
foreach (var key in toInsert)
{
// and add a new one
var node = _indexer.AddNode(index, key, pkNode);
// link my node to data block
node.DataBlock = dataBlock.Position;
}
}
return(true);
}