/// <summary>
/// Copy all file content to a steam
/// </summary>
public void Download(string id, Stream stream)
{
if (stream == null)
{
throw new ArgumentNullException("stream");
}
var file = this.FindById(id);
if (file == null)
{
throw LiteException.FileNotFound(id);
}
file.CopyTo(stream);
}
/// <summary>
/// Drop index and release slot for another index
/// </summary>
public bool DropIndex(string field)
{
if (string.IsNullOrEmpty(field))
{
throw new ArgumentNullException("field");
}
if (field == "_id")
{
throw LiteException.IndexDropId();
}
// start transaction
this.Database.Transaction.Begin();
try
{
var col = this.GetCollectionPage(false);
// if collection not exists, no drop
if (col == null)
{
this.Database.Transaction.Abort();
return(false);
}
// search for index reference
var index = col.GetIndex(field);
// delete all data pages + indexes pages
this.Database.Indexer.DropIndex(index);
// clear index reference
index.Clear();
// save collection page
col.IsDirty = true;
this.Database.Transaction.Commit();
return(true);
}
catch
{
this.Database.Transaction.Rollback();
throw;
}
}
/// <summary>
/// Drop an index from a collection
/// </summary>
public bool DropIndex(string collection, string field)
{
if (collection.IsNullOrWhiteSpace())
{
throw new ArgumentNullException("collection");
}
if (field.IsNullOrWhiteSpace())
{
throw new ArgumentNullException("field");
}
if (field == "_id")
{
throw LiteException.IndexDropId();
}
return(this.Transaction <bool>(collection, false, (col) =>
{
// no collection, no index
if (col == null)
{
return false;
}
// search for index reference
var index = col.GetIndex(field);
// no index, no drop
if (index == null)
{
return false;
}
_log.Write(LoggerWrap.COMMAND, "drop index on '{0}' :: '{1}'", collection, field);
// delete all data pages + indexes pages
_indexer.DropIndex(index);
// clear index reference
index.Clear();
// mark collection page as dirty
_pager.SetDirty(col);
return true;
}));
}
/// <summary>
/// Create a new permanent index in all documents inside this collections if index not exists already. Returns true if index was created or false if already exits
/// </summary>
/// <param name="field">Document field name (case sensitive)</param>
/// <param name="unique">If is a unique index</param>
public bool EnsureIndex(string field, bool unique = false)
{
if (string.IsNullOrEmpty(field))
{
throw new ArgumentNullException("field");
}
if (field == "_id")
{
return(false); // always exists
}
if (!CollectionIndex.IndexPattern.IsMatch(field))
{
throw LiteException.InvalidFormat("IndexField", field);
}
return(_engine.Value.EnsureIndex(_name, field, unique));
}
/// <summary>
/// Get a field name passing mapper type and returns property type
/// </summary>
private string GetTypeField(Type type, string property, out Type propertyType)
{
// lets get mapping bettwen .NET class and BsonDocument
var map = _mapper.GetPropertyMapper(type);
PropertyMapper prop;
if (map.TryGetValue(property, out prop))
{
propertyType = prop.PropertyType;
return(prop.FieldName);
}
else
{
throw LiteException.PropertyNotMapped(property);
}
}
/// <summary>
/// Set datafile length
/// </summary>
public void SetLength(long fileSize)
{
// if readonly, do nothing
if (_options.ReadOnly)
{
return;
}
// checks if new fileSize will exceed limit size
if (fileSize > _options.LimitSize)
{
throw LiteException.FileSizeExceeded(_options.LimitSize);
}
// fileSize parameter tell me final size of data file - helpful to extend first datafile
_stream.SetLength(fileSize);
}
internal LiteFileInfo(LiteEngine engine, string id, string filename)
{
if (!IdPattern.IsMatch(id))
{
throw LiteException.InvalidFormat("FileId", id);
}
_engine = engine;
this.Id = id;
this.Filename = Path.GetFileName(filename);
this.MimeType = MimeTypeConverter.GetMimeType(this.Filename);
this.Length = 0;
this.Chunks = 0;
this.UploadDate = DateTime.Now;
this.Metadata = new BsonDocument();
}
/// <summary>
/// Initialize LiteEngine using custom disk service implementation and full engine options
/// </summary>
public LiteEngine(IDiskService disk, string password = null, TimeSpan?timeout = null, int cacheSize = 5000, Logger log = null)
{
if (disk == null)
{
throw new ArgumentNullException("disk");
}
_timeout = timeout ?? TimeSpan.FromMinutes(1);
_cacheSize = cacheSize;
_disk = disk;
_log = log ?? new Logger();
// initialize datafile (create) and set log instance
_disk.Initialize(_log, password);
// read header page
var header = BasePage.ReadPage(_disk.ReadPage(0)) as HeaderPage;
// hash password with sha1 or keep as empty byte[20]
var sha1 = password == null ? new byte[20] : AesEncryption.HashSHA1(password);
// compare header password with user password even if not passed password (datafile can have password)
if (sha1.BinaryCompareTo(header.Password) != 0)
{
// explicit dispose
_disk.Dispose();
throw LiteException.DatabaseWrongPassword();
}
// initialize AES encryptor
if (password != null)
{
_crypto = new AesEncryption(password, header.Salt);
}
// initialize all services
this.InitializeServices();
// try recovery data only if journal are enabled
if (_disk.IsJournalEnabled)
{
// try recovery if has journal file
_trans.Recovery();
}
}
/// <summary>
/// Copy all file content to a steam
/// </summary>
public LiteFileInfo Download(string id, Stream stream)
{
if (stream == null)
{
throw new ArgumentNullException(nameof(stream));
}
var file = this.FindById(id);
if (file == null)
{
throw LiteException.FileNotFound(id);
}
file.CopyTo(stream);
return(file);
}
/// <summary>
/// Update a document in this collection. Returns false if not found document in collection
/// </summary>
public virtual bool Update(T document)
{
if (document == null)
{
throw new ArgumentNullException("document");
}
// get BsonDocument from object
var doc = this.Database.Mapper.ToDocument(document);
var id = doc["_id"];
if (id.IsNull || id.IsMinValue || id.IsMaxValue)
{
throw LiteException.InvalidDataType("_id", id);
}
return(this.UpdateDoc(id, doc));
}
/// <summary>
/// Try execute a block of code until timeout when IO lock exception occurs
/// </summary>
public void TryExec(Action action)
{
var timeout = DateTime.Now.Add(_connectionString.Timeout);
while (DateTime.Now < timeout)
{
try
{
action();
return;
}
catch (IOException ex)
{
ex.WaitIfLocked(250);
}
}
throw LiteException.LockTimeout(_connectionString.Timeout);
}
/// <summary>
/// Rename a collection
/// </summary>
public bool RenameCollection(string collection, string newName)
{
if (collection.IsNullOrWhiteSpace())
{
throw new ArgumentNullException("collection");
}
if (newName.IsNullOrWhiteSpace())
{
throw new ArgumentNullException("newName");
}
return(this.Transaction <bool>(collection, false, (col) =>
{
if (col == null)
{
return false;
}
_log.Write(LoggerWrap.COMMAND, "rename collection '{0}' -> '{1}'", collection, newName);
// check if newName already exists
if (this.GetCollectionNames().Contains(newName, StringComparer.OrdinalIgnoreCase))
{
throw LiteException.AlreadyExistsCollectionName(newName);
}
// change collection name and save
col.CollectionName = newName;
// set collection page as dirty
_pager.SetDirty(col);
// update header collection reference
var header = _pager.GetPage <HeaderPage>(0);
header.CollectionPages.Remove(collection);
header.CollectionPages.Add(newName, col.PageID);
_pager.SetDirty(header);
return true;
}));
}
/// <summary>
/// Persist single page bytes to disk
/// </summary>
public virtual void WritePage(uint pageID, byte[] buffer)
{
if (_options.ReadOnly)
{
throw LiteException.ReadOnlyDatabase();
}
var position = BasePage.GetSizeOfPages(pageID);
_log.Write(Logger.DISK, "write page #{0:0000} :: {1}", pageID, (PageType)buffer[PAGE_TYPE_POSITION]);
// position cursor
if (_stream.Position != position)
{
_stream.Seek(position, SeekOrigin.Begin);
}
_stream.Write(buffer, 0, BasePage.PAGE_SIZE);
}
/// <summary>
/// Test if cache still valid (if datafile was changed by another process reset cache)
/// Returns true if file was changed
/// [Thread Safe]
/// </summary>
private bool DetectDatabaseChanges()
{
// if disk are exclusive don't need check dirty read
if (_disk.IsExclusive)
{
return(false);
}
// empty cache? just exit
if (_cache.CleanUsed == 0)
{
return(false);
}
_log.Write(Logger.CACHE, "checking disk to detect database changes from another process");
// get ChangeID from cache
var header = _cache.GetPage(0) as HeaderPage;
var changeID = header == null ? 0 : header.ChangeID;
// and get header from disk
var disk = BasePage.ReadPage(_disk.ReadPage(0)) as HeaderPage;
// if disk header are in recovery mode, throw exception to datafile re-open and recovery pages
if (disk.Recovery)
{
_log.Write(Logger.ERROR, "datafile in recovery mode, need re-open database");
throw LiteException.NeedRecover();
}
// if header change, clear cache and add new header to cache
if (disk.ChangeID != changeID)
{
_log.Write(Logger.CACHE, "file changed from another process, cleaning all cache pages");
_cache.ClearPages();
_cache.AddPage(disk);
return(true);
}
return(false);
}
/// <summary>
/// Create a new instance from a Type
/// </summary>
public static object CreateInstance(Type type)
{
try
{
CreateObject c = null;
if (_cacheCtor.TryGetValue(type, out c))
{
return(c());
}
else
{
if (type.IsClass)
{
var dynMethod = new DynamicMethod("_", type, null);
var il = dynMethod.GetILGenerator();
il.Emit(OpCodes.Newobj, type.GetConstructor(Type.EmptyTypes));
il.Emit(OpCodes.Ret);
c = (CreateObject)dynMethod.CreateDelegate(typeof(CreateObject));
_cacheCtor.Add(type, c);
}
else // structs
{
var dynMethod = new DynamicMethod("_", typeof(object), null);
var il = dynMethod.GetILGenerator();
var lv = il.DeclareLocal(type);
il.Emit(OpCodes.Ldloca_S, lv);
il.Emit(OpCodes.Initobj, type);
il.Emit(OpCodes.Ldloc_0);
il.Emit(OpCodes.Box, type);
il.Emit(OpCodes.Ret);
c = (CreateObject)dynMethod.CreateDelegate(typeof(CreateObject));
_cacheCtor.Add(type, c);
}
return(c());
}
}
catch (Exception)
{
throw LiteException.InvalidCtor(type);
}
}
/// <summary>
/// Try execute some action while has lock exception
/// </summary>
public static void TryExec(Action action, TimeSpan timeout)
{
var timer = DateTime.UtcNow.Add(timeout);
do
{
try
{
action();
return;
}
catch (IOException ex)
{
ex.WaitIfLocked(25);
}
}while (DateTime.UtcNow < timer);
throw LiteException.LockTimeout(timeout);
}
private static PropertyInfo SelectProperty(IEnumerable <PropertyInfo> props, params Func <PropertyInfo, bool>[] predicates)
{
foreach (var predicate in predicates)
{
var prop = props.FirstOrDefault(predicate);
if (prop != null)
{
if (!prop.CanRead || !prop.CanWrite)
{
throw LiteException.PropertyReadWrite(prop);
}
return(prop);
}
}
return(null);
}
/// <summary>
/// Add a new collection. Check if name the not exists
/// </summary>
public CollectionPage Add(string name)
{
if (string.IsNullOrEmpty(name))
{
throw new ArgumentNullException(nameof(name));
}
if (!CollectionPage.NamePattern.IsMatch(name))
{
throw LiteException.InvalidFormat(name);
}
_log.Write(Logger.COMMAND, "creating new collection '{0}'", name);
// get header marked as dirty because I will use header after (and NewPage can get another header instance)
var header = _pager.GetPage <HeaderPage>(0);
// check limit count (8 bytes per collection = 4 to string length, 4 for uint pageID)
if (header.CollectionPages.Sum(x => x.Key.Length + 8) + name.Length + 8 >= CollectionPage.MAX_COLLECTIONS_SIZE)
{
throw LiteException.CollectionLimitExceeded(CollectionPage.MAX_COLLECTIONS_SIZE);
}
// get new collection page (marked as dirty)
var col = _pager.NewPage <CollectionPage>();
// add this page to header page collection
header.CollectionPages.Add(name, col.PageID);
col.CollectionName = name;
// set header page as dirty
_pager.SetDirty(header);
// create PK index
var pk = _indexer.CreateIndex(col);
pk.Field = "_id";
pk.Expression = "$._id";
pk.Unique = true;
return(col);
}
/// <summary>
/// Override read page decrypting data from disk
/// </summary>
public override byte[] ReadPage(uint pageID)
{
var buffer = base.ReadPage(pageID);
// when read header, checks passoword
if (pageID == 0)
{
// I know, header page will be double read (it's the price for isolated concerns)
var header = (HeaderPage)BasePage.ReadPage(buffer);
if (header.DbParams.Password.BinaryCompareTo(_password) != 0)
{
throw LiteException.DatabaseWrongPassword();
}
return(buffer);
}
return(_crypto.Decrypt(buffer));
}
/// <summary>
/// Lock datafile agains other process read/write
/// </summary>
public void Lock()
{
#if NETFULL
this.TryExec(() =>
{
_lockLength = _stream.Length;
_log.Write(Logger.DISK, "lock datafile");
_stream.Lock(0, _lockLength);
});
#elif NETCORE
bool locked = _lockSemaphore.Wait(_timeout);
if (!locked)
{
LiteException.LockTimeout(_timeout);
}
_lockLength = _stream.Length;
_log.Write(Logger.DISK, "lock datafile");
#endif
}
public IBsonDataReader Execute()
{
var first = _tokenizer.ReadToken().Expect(TokenType.Word);
LOG($"executing `{first.Value.ToUpper()}`", "SQL");
switch (first.Value.ToUpper())
{
case "SELECT": return(this.ParseSelect(false));
case "EXPLAIN":
_tokenizer.ReadToken().Expect("SELECT");
return(this.ParseSelect(true));
case "INSERT": return(this.ParseInsert());
case "DELETE": return(this.ParseDelete());
case "UPDATE": return(this.ParseUpadate());
case "DROP": return(this.ParseDrop());
case "RENAME": return(this.ParseRename());
case "CREATE": return(this.ParseCreate());
case "ANALYZE": return(this.ParseAnalyze());
case "CHECKPOINT": return(this.ParseCheckpoint());
case "VACCUM": return(this.ParseVaccum());
case "BEGIN": return(this.ParseBegin());
case "ROLLBACK": return(this.ParseRollback());
case "COMMIT": return(this.ParseCommit());
default: throw LiteException.UnexpectedToken(first);
}
}
/// <summary>
/// Create a new permanent index in all documents inside this collections if index not exists already. Returns true if
/// index was created or false if already exits
/// </summary>
/// <param name="field">Document field name (case sensitive)</param>
/// <param name="options">All index options</param>
public bool EnsureIndex(string field, IndexOptions options)
{
if (string.IsNullOrEmpty(field))
{
throw new ArgumentNullException("field");
}
if (options == null)
{
throw new ArgumentNullException("options");
}
if (field == "_id")
{
return(false); // always exists
}
if (!CollectionIndex.IndexPattern.IsMatch(field))
{
throw LiteException.InvalidFormat("IndexField", field);
}
return(_engine.EnsureIndex(Name, field, options));
}
/// <summary>
/// Read a page with correct instance page object. Checks for pageType
/// </summary>
public static BasePage ReadPage(byte[] buffer)
{
var reader = new ByteReader(buffer);
var pageID = reader.ReadUInt32();
var pageType = (PageType)reader.ReadByte();
if (pageID == 0 && (byte)pageType > 5)
{
throw LiteException.InvalidDatabase();
}
var page = CreateInstance(pageID, pageType);
page.ReadHeader(reader);
page.ReadContent(reader);
page.DiskData = buffer;
return(page);
}
/// <summary>
/// Get a bson string field based on class PropertyInfo using BsonMapper class
/// Support Name1.Name2 dotted notation
/// </summary>
private string GetBsonField(string property)
{
var parts = property.Split('.');
var fields = new string[parts.Length];
var type = _type;
var isdbref = false;
PropertyMapper prop;
// loop "first.second.last"
for (var i = 0; i < parts.Length; i++)
{
var map = _mapper.GetPropertyMapper(type);
var part = parts[i];
if (map.TryGetValue(part, out prop))
{
type = prop.PropertyType;
fields[i] = prop.FieldName;
if (prop.FieldName == "_id" && isdbref)
{
isdbref = false;
fields[i] = "$id";
}
// if this property is DbRef, so if next property is _id, change to $id
if (prop.IsDbRef)
{
isdbref = true;
}
}
else
{
throw LiteException.PropertyNotMapped(property);
}
}
return(string.Join(".", fields));
}
/// <summary>
/// SHRINK
/// </summary>
private BsonDataReader ParseRebuild()
{
_tokenizer.ReadToken().Expect("REBUILD");
var options = new RebuildOptions();
// read <eol> or ;
var next = _tokenizer.LookAhead();
if (next.Type == TokenType.EOF || next.Type == TokenType.SemiColon)
{
options = null;
_tokenizer.ReadToken();
}
else
{
var reader = new JsonReader(_tokenizer);
var json = reader.Deserialize();
if (json.IsDocument == false)
{
throw LiteException.UnexpectedToken(next);
}
if (json["password"].IsString)
{
options.Password = json["password"];
}
if (json["collation"].IsString)
{
options.Collation = new Collation(json["collation"].AsString);
}
}
var diff = _engine.Rebuild(options);
return(new BsonDataReader((int)diff));
}
/// <summary>
/// Based on an expression, returns document field mapped from class Property.
/// Support multi level dotted notation: x => x.Customer.Name
/// Prefix is used on array expression like: x => x.Customers.Any(z => z.Name == "John") (prefix = "Customers."
/// </summary>
public string GetField(Expression expr, string prefix = "")
{
var property = prefix + expr.GetPath();
var parts = property.Split('.');
var fields = new string[parts.Length];
var type = _type;
var isdbref = false;
// loop "first.second.last"
for (var i = 0; i < parts.Length; i++)
{
var entity = _mapper.GetEntityMapper(type);
var part = parts[i];
var prop = entity.Members.Find(x => x.MemberName == part);
if (prop == null)
{
throw LiteException.PropertyNotMapped(property);
}
// if property is an IEnumerable, gets underlying type (otherwise, gets PropertyType)
type = prop.UnderlyingType;
fields[i] = prop.FieldName;
if (prop.FieldName == "_id" && isdbref)
{
isdbref = false;
fields[i] = "$id";
}
// if this property is DbRef, so if next property is _id, change to $id
if (prop.IsDbRef)
{
isdbref = true;
}
}
return(string.Join(".", fields));
}
/// <summary>
/// Enter in Exclusive lock mode
/// </summary>
public LockControl Write()
{
// if already in exclusive, do nothing
if (_thread.IsWriteLockHeld)
{
return(new LockControl(() => { }));
}
// let's test if is not in read lock
if (_thread.IsReadLockHeld)
{
throw new NotSupportedException("Not support Write lock inside a Read lock");
}
// try enter in write mode (thread)
if (!_thread.TryEnterWriteLock(_timeout))
{
throw LiteException.LockTimeout(_timeout);
}
_log.Write(Logger.LOCK, "entered in write lock mode in thread #{0}", Thread.CurrentThread.ManagedThreadId);
// try enter in exclusive mode in disk
var position = _disk.Lock(LockState.Write, _timeout);
// call avoid dirty only if not came from a shared mode
this.DetectDatabaseChanges();
return(new LockControl(() =>
{
// release disk write
_disk.Unlock(LockState.Write, position);
// release thread write
_thread.ExitWriteLock();
_log.Write(Logger.LOCK, "exited write lock mode in thread #{0}", Thread.CurrentThread.ManagedThreadId);
}));
}
protected override void ReadContent(ByteReader reader)
{
var info = reader.ReadString(HEADER_INFO.Length);
var ver = reader.ReadByte();
if (info != HEADER_INFO) throw LiteException.InvalidDatabase();
if (ver != FILE_VERSION) throw LiteException.InvalidDatabaseVersion(ver);
this.ChangeID = reader.ReadUInt16();
this.FreeEmptyPageID = reader.ReadUInt32();
this.LastPageID = reader.ReadUInt32();
this.UserVersion = reader.ReadUInt16();
this.Password = reader.ReadBytes(this.Password.Length);
this.Salt = reader.ReadBytes(this.Salt.Length);
// read page collections references (position on end of page)
var cols = reader.ReadByte();
for (var i = 0; i < cols; i++)
{
this.CollectionPages.Add(reader.ReadString(), reader.ReadUInt32());
}
}
/// <summary>
/// Try run an operation over datafile - keep tring if locked
/// </summary>
private void TryExec(Action action)
{
var timer = DateTime.UtcNow.Add(_options.Timeout);
var locked = false;
while (DateTime.UtcNow < timer)
{
try
{
action();
return;
}
catch (UnauthorizedAccessException)
{
// this exception can occurs when a single instance release file but Windows Filesystem are not fully released.
if (locked == false)
{
locked = true;
_log.Write(Logger.ERROR, "unauthorized file access, waiting for {0} timeout", _options.Timeout);
}
IOExceptionExtensions.WaitFor(250);
}
catch (IOException ex)
{
if (locked == false)
{
locked = true;
_log.Write(Logger.ERROR, "locked file, waiting for {0} timeout", _options.Timeout);
}
ex.WaitIfLocked(250);
}
}
_log.Write(Logger.ERROR, "timeout disk access after {0}", _options.Timeout);
throw LiteException.LockTimeout(_options.Timeout);
}
public override void ReadContent(BinaryReader reader)
{
var info = reader.ReadString();
if (info != HEADER_INFO)
{
throw LiteException.InvalidDatabase(reader.BaseStream);
}
var ver = reader.ReadByte();
if (ver != FILE_VERSION)
{
throw LiteException.InvalidDatabaseVersion(reader.BaseStream, ver);
}
this.ChangeID = reader.ReadUInt16();
this.FreeEmptyPageID = reader.ReadUInt32();
this.FirstCollectionPageID = reader.ReadUInt32();
this.LastPageID = reader.ReadUInt32();
this.UserVersion = reader.ReadInt32();
}
