Example #1
0
        public TextDeferredIndexer(DBreezeEngine engine)
        {
            this.DBreezeEngine = engine;
            LTrieSettings = new TrieSettings()
            {
                InternalTable = true
            };
            Storage = new StorageLayer(Path.Combine(engine.MainFolder, TableFileName), LTrieSettings, engine.Configuration);
            LTrie = new LTrie(Storage);
            LTrie.TableName = "DBreeze.TextIndexer";

            if (LTrie.Storage.Length > 100000)  //Recreating file if its size more then 100KB and it is empty
            {
                if (LTrie.Count(true) == 0)
                {
                    LTrie.Storage.RecreateFiles();
                    LTrie.Dispose();

                    Storage = new StorageLayer(Path.Combine(engine.MainFolder, TableFileName), LTrieSettings, engine.Configuration);
                    LTrie = new LTrie(Storage);
                    LTrie.TableName = "DBreeze.TextIndexer";
                }
            }

            if (LTrie.Count(true) > 0)
                this.StartDefferedIndexing();
        }
Example #2
0
        public StorageLayer(string fileName, TrieSettings trieSettings, DBreezeConfiguration configuration)
        {
            if (trieSettings.StorageWasOverriden)
            {
                switch (trieSettings.AlternativeTableStorageType)
                {
                    case DBreezeConfiguration.eStorage.DISK:

                        _tableStorage = (IStorage) new FSR(fileName, trieSettings, configuration);

                        break;
                    case DBreezeConfiguration.eStorage.MEMORY:

                        _tableStorage = (IStorage)new MSR(fileName, trieSettings, configuration);

                        break;
                    case DBreezeConfiguration.eStorage.RemoteInstance:

                        _tableStorage = (IStorage)new RISR(fileName, trieSettings, configuration);

                        break;
                }
            }
            else
            {
                switch (configuration.Storage)
                {
                    case DBreezeConfiguration.eStorage.DISK:

                        _tableStorage = (IStorage)new FSR(fileName, trieSettings, configuration);

                        break;
                    case DBreezeConfiguration.eStorage.MEMORY:

                        _tableStorage = (IStorage)new MSR(fileName, trieSettings, configuration);

                        break;
                    case DBreezeConfiguration.eStorage.RemoteInstance:

                        _tableStorage = (IStorage)new RISR(fileName, trieSettings, configuration);

                        break;
                }
            }
        }
Example #3
0
        public RISR(string fileName, TrieSettings trieSettings, DBreezeConfiguration configuration)
        {
            this._fileName = fileName;
            this._configuration = configuration;
            this._trieSettings = trieSettings;
            DefaultPointerLen = this._trieSettings.POINTER_LENGTH;

            _backupIsActive = this._configuration.Backup.IsActive;

            //Transforms fileName into ulong digit
            ulFileName = this._configuration.Backup.BackupFNP.ParseFilename(Path.GetFileNameWithoutExtension(this._fileName));

            //Setting up RemoteCommander
            RIC = new DBreeze.Storage.RemoteInstance.RemoteInstanceCommander(configuration.RICommunicator);

            InitFiles();
        }
Example #4
0
        /*          TODO

         *  1. HERE we will add TableNames as RegEx with settings
         *  2. Checking Reserverd TableNames prefixes
         *  3. User TableName must start from @ut
         *  4. GetPhysicalPathToTheUserTable - File with DIrectory Settings for different tables parser (to make reside different tables in different HDDs or even network drives)
         */
        private void OpenSchema()
        {
            LTrieSettings = new TrieSettings()
            {
                InternalTable = true,
                //SkipStorageBuffer = true
            };

            Storage = new StorageLayer(Path.Combine(Engine.MainFolder, SchemaFileName), LTrieSettings, Engine.Configuration);

            LTrie = new LTrie(Storage);

            LTrie.TableName = "DBreeze.Scheme";

            //Reading lastFileNumber
            ReadUserLastFileNumber();
        }
Example #5
0
        /// <summary>
        /// Returns table for READ, WRITE FUNC
        /// </summary>
        /// <param name="userTableName"></param>
        /// <returns></returns>
        internal LTrie GetTable(string userTableName)
        {
            string tableName = GetUserTableNameAsString(userTableName);

            //TODO pattern based mapping If table doesn't exist we create it with properties which could be supplied after db init as regex theme.

            //Schema protocol: 2 bytes - protocol version, other data
            //For protocol 1: first 8 bytes will be TheFileName, starting from db10000-dbN (0-N ulong). up to 10000 are reserved for dbreeze.

            //Table names are UTF-8 based, no limits

            ulong fileName = 0;
            OpenTable otl = null;

            _sync_openTablesHolder.EnterUpgradeableReadLock();
            try
            {

                _openTablesHolder.TryGetValue(tableName, out otl);

                if (otl != null)
                {
                    //Try to increase usage and return LTrie
                    otl.Add();
                    return otl.Trie;
                }

                //Probably table Exists in db but not in openTablesHolder

                _sync_openTablesHolder.EnterWriteLock();
                try
                {
                    //UpgradeableRead recheck
                    _openTablesHolder.TryGetValue(tableName, out otl);

                    if (otl != null)
                    {
                        //Try to increase usage and return LTrie
                        otl.Add();
                        return otl.Trie;
                    }

                    byte[] btTableName = GetUserTableNameAsByte(userTableName);

                    //Trying to get fileName from cache
                    fileName = this.cachedTableNames.GetFileName(tableName);
                    // LTrieRow row = null;
                    bool tableExists = false;

                    if (fileName == 0)
                    {
                        LTrieRow row = LTrie.GetKey(btTableName, false);

                        if (row.Exists)
                        {
                            tableExists = true;

                            byte[] fullValue = row.GetFullValue(false);
                            //Can be parsed different. First protocol version is 1
                            ushort schemeProtocol = fullValue.Substring(0, 2).To_UInt16_BigEndian();

                            switch (schemeProtocol)
                            {
                                case 1:
                                    fileName = fullValue.Substring(2, 8).To_UInt64_BigEndian();
                                    break;
                                default:
                                    throw DBreezeException.Throw(DBreezeException.eDBreezeExceptions.SCHEME_FILE_PROTOCOL_IS_UNKNOWN);
                            }
                        }
                        else
                        {
                            tableExists = false;
                            //Creating new table.

                            //Checking table name validity

                            //this will throw exception, if not valid
                            DbUserTables.UserTableNameIsOk(userTableName);

                            //Creating such table and renewing LastFileNumber counter

                            //Adding to LastFileNumber
                            LastFileNumber++;

                            ////Deleting physical files related to the table, if they existed - normally they should not
                            //DeleteAllReleatedTableFiles(Path.Combine(Engine.MainFolder, LastFileNumber.ToString()));

                            byte[] lft = LastFileNumber.To_8_bytes_array_BigEndian();

                            //Writing this number to Schema file
                            LTrie.Add(Encoding.UTF8.GetBytes(LastFileNumberKeyName), lft);

                            //Creating table self and writing to Schema file

                            LTrie.Add(btTableName,
                                new byte[] { 0, 1 }     //Protocol version 1
                                .Concat(lft));          //Number of the file

                            //Committing both records
                            LTrie.Commit();

                            fileName = LastFileNumber;

                            this.cachedTableNames.Add(tableName, fileName);
                        }
                    }
                    else
                        tableExists = true;

                    //Creating LTrie, adding it to _openTablesHolder

                    //Seeting up Trie TableName, OTHER SETTINGS

                    TrieSettings ts = new TrieSettings();
                    IStorage storage = null;

                    ////Checking if default Flusg Disk behaviour was overriden
                    //ts.DiskFlushBehaviour = Engine.Configuration.DiskFlushBehaviour;
                    ////Checking if we have alternative DiskFlush behaviour
                    //foreach (var pattern in Engine.Configuration.AlternativeDiskFlushBehaviour)
                    //{
                    //    //pattern.Key
                    //    if (DbUserTables.PatternsIntersect(pattern.Key, userTableName))
                    //    {

                    //        ts.DiskFlushBehaviour = pattern.Value;
                    //        break;
                    //    }
                    //}

                    string alternativeTableLocation = String.Empty;

                    if (CheckAlternativeTableLocationsIntersections(userTableName, out alternativeTableLocation))
                    {
                        ts.StorageWasOverriden = true;

                        if (alternativeTableLocation == String.Empty)
                        {
                            ts.AlternativeTableStorageType = DBreezeConfiguration.eStorage.MEMORY;

                            storage = new StorageLayer(Path.Combine(Engine.MainFolder, fileName.ToString()), ts, Engine.Configuration);
                        }
                        else
                        {
                            ts.AlternativeTableStorageType = DBreezeConfiguration.eStorage.DISK;
                            ts.AlternativeTableStorageFolder = alternativeTableLocation;

                            DirectoryInfo diAlt = new DirectoryInfo(alternativeTableLocation);
                            if (!diAlt.Exists)
                                diAlt.Create();

                            if (!tableExists)
                            {
                                //Deleting physical files related to the table, if they existed - normally they should not
                                DeleteAllReleatedTableFiles(Path.Combine(ts.AlternativeTableStorageFolder, LastFileNumber.ToString()));
                            }

                            storage = new StorageLayer(Path.Combine(ts.AlternativeTableStorageFolder, fileName.ToString()), ts, Engine.Configuration);
                        }
                    }
                    else
                    {
                        if (!tableExists)
                        {
                            //Deleting physical files related to the table, if they existed - normally they should not
                            DeleteAllReleatedTableFiles(Path.Combine(Engine.MainFolder, LastFileNumber.ToString()));
                        }

                        storage = new StorageLayer(Path.Combine(Engine.MainFolder, fileName.ToString()), ts, Engine.Configuration);
                    }

                    //storage = new StorageLayer(Path.Combine(Engine.MainFolder, fileName.ToString()), ts, Engine.Configuration);

                    LTrie trie = new LTrie(storage);

                    //Setting LTrie user table name
                    trie.TableName = userTableName;

                    //_openTablesHolder.Add(tableName, trie);

                    //Automatically increased usage in OpenTable constructor
                    _openTablesHolder.Add(tableName, new OpenTable(trie));

                    return trie;
                }
                catch (System.Exception ex)
                {
                    //CASCADE
                    throw ex;
                }
                finally
                {
                    _sync_openTablesHolder.ExitWriteLock();
                }
            }
            catch (Exception ex)
            {
                throw DBreezeException.Throw(DBreezeException.eDBreezeExceptions.SCHEME_GET_TABLE_WRITE_FAILED, tableName, ex);
            }
            finally
            {
                _sync_openTablesHolder.ExitUpgradeableReadLock();
            }
        }
Example #6
0
 public NestedTableStorage(IStorage masterStorage, TrieSettings trieSettings)
 {
     _masterStorage = masterStorage;
     _trieSettings = trieSettings;
 }
Example #7
0
        private void RestoreNotFinishedTransactions()
        {
            //TODO Trie settings from the table must be taken from schema (when they will differ)

            //STORE FILE NAME of rollback not table name
            try
            {
                byte[] btCommittedTablesNames =null;
                List<string> committedTablesNames = new List<string>();

                if (LTrie.Count(false) == 0)     //All ok
                {
                    LTrie.RemoveAll(true);
                    return;
                }

                string physicalPathToTheUserTable = String.Empty;

                //Settigns and storage for Committed tables !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!   MUST BE TAKEN FROM SCHEMA, FOR NOW DEFAULT
                TrieSettings ltrSet = null;
                IStorage storage = null;
                DBreeze.LianaTrie.LTrie ltrie = null;

                foreach (var row in LTrie.IterateForward())
                {
                    btCommittedTablesNames = row.GetFullValue(true);

                    committedTablesNames = System.Text.Encoding.UTF8.GetString(btCommittedTablesNames).DeserializeXml<List<string>>();

                    foreach (var fn in committedTablesNames)
                    {
                        //Trying to get path from the Schema, there is universal function for getting table physical TABLE FULL PATH /NAME

                        physicalPathToTheUserTable = Engine.DBreezeSchema.GetPhysicalPathToTheUserTable(fn);

                        //Returned path can be empty, if no more such table
                        if (physicalPathToTheUserTable == String.Empty)
                            continue;

                        //We don't restore in-memory tables
                        if (physicalPathToTheUserTable == "MEMORY")
                            continue;

                        //we open ltrie, and it automatically restores rollback
                        ltrSet = new TrieSettings();     //!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!   MUST BE TAKEN FROM SCHEMA, FOR NOW DEFAULT
                        //storage = new TrieDiskStorage(physicalPathToTheUserTable, ltrSet, Engine.Configuration);
                        storage = new StorageLayer(physicalPathToTheUserTable, ltrSet, Engine.Configuration);
                        ltrie = new LTrie(storage);

                        //closing trie, that Schema could open it again
                        ltrie.Dispose();

                        ////Deleting rollback file for such table
                        //physicalPathToTheUserTable += ".rol";
                        //System.IO.File.Delete(physicalPathToTheUserTable);
                    }

                    committedTablesNames.Clear();
                }

                //If all ok, recreate file
                LTrie.RemoveAll(true);
            }
            catch (OperationCanceledException ex)
            {
                throw ex;
            }
            //catch (System.Threading.ThreadAbortException ex)
            //{
            //    //We don'T make DBisOperable = false;
            //    throw ex;
            //}
            catch (Exception ex)
            {
                //BRINGS TO DB NOT OPERATABLE
                this.Engine.DBisOperable = false;
                this.Engine.DBisOperableReason = "TransactionsCoordinator.RestoreNotFinishedTransaction";
                //NOT CASCADE ADD EXCEPTION
                throw DBreezeException.Throw(DBreezeException.eDBreezeExceptions.CLEAN_ROLLBACK_FILES_FOR_FINISHED_TRANSACTIONS_FAILED);
            }
        }
Example #8
0
        private void Init()
        {
            try
            {
                LTrieSettings = new TrieSettings()
                {
                     InternalTable = true,
                     //SkipStorageBuffer = true
                };

                Storage = new StorageLayer(Path.Combine(Engine.MainFolder, JournalFileName), LTrieSettings, Engine.Configuration);
                 //Storage = new TrieDiskStorage(Path.Combine(Engine.MainFolder, JournalFileName), LTrieSettings, Engine.Configuration);
                 LTrie = new LTrie(Storage);

                 LTrie.TableName = "DBreeze.TranJournal";

                 this.RestoreNotFinishedTransactions();
            }
            catch (Exception ex)
            {
                //CASCADE
                throw ex;
            }
        }
Example #9
0
        public FSR(string fileName, TrieSettings trieSettings, DBreezeConfiguration configuration)
        {
            this._fileName = fileName;
            this._configuration = configuration;
            this._trieSettings = trieSettings;
            DefaultPointerLen = this._trieSettings.POINTER_LENGTH;

            _backupIsActive = this._configuration.Backup.IsActive;

            //Transforms fileName into ulong digit
            ulFileName = this._configuration.Backup.BackupFNP.ParseFilename(Path.GetFileNameWithoutExtension(this._fileName));

            InitFiles();
        }
Example #10
0
        public MSR(string fileName, TrieSettings trieSettings,DBreezeConfiguration configuration)
        {
            this._fileName = fileName;
            this._configuration = configuration;
            this._trieSettings = trieSettings;
            DefaultPointerLen = this._trieSettings.POINTER_LENGTH;

            InitFiles();
        }
Example #11
0
        private void TestKrome()
        {
            using (var tran = engine.GetTransaction())
            {
                byte[] ptr = tran.InsertDataBlock("t1", null, new byte[] { 1, 2, 3 });

                tran.Insert<int, byte[]>("t1", 1, ptr);
                //NestedTable nt = tran.InsertTable<int>("t1", 1, 0);
                //nt.Insert<int, int>(1, 1);
                tran.Commit();
            }

            using (var tran = engine.GetTransaction())
            {
                var row = tran.Select<int,byte[]>("t1",1);
                byte[] val = row.Value;

                //NestedTable nt = tran.SelectTable<int>("t1", 1, 0);
                //var row = nt.Select<int, int>(1);

                tran.RemoveAllKeys("t1", true);

                byte[] res = tran.SelectDataBlock("t1", val);

                //Console.WriteLine("Key: {0}", row.Value);
            }
            return;

            //using (var tran = engine.GetTransaction())
            //{
            //    for (int i = 0; i < 1000000; i++)
            //    {
            //        tran.Insert<int, byte>("t1", i, 1);
            //    }

            //    tran.Commit();
            //}

            //Console.WriteLine("***");
            //byte bt = 1;

            //using (var tran = engine.GetTransaction())
            //{
            //    DBreeze.Diagnostic.SpeedStatistic.StartCounter("a");
            //    foreach (var row in tran.SelectForward<int, byte>("t1"))
            //    {
            //        bt = row.Value;
            //       // Console.WriteLine("Key: {0}", row.Key);
            //    }
            //    DBreeze.Diagnostic.SpeedStatistic.PrintOut("a",true);
            //}

            //return;

            using (var tran = engine.GetTransaction())
            {
                tran.Insert<int, byte>("t1", 1, 1);
                tran.Insert<int, byte>("t2", 1, 1);
                tran.Insert<int, byte>("t3", 1, 1);

                tran.Commit();
            }

            using (var tran = engine.GetTransaction())
            {
                tran.SynchronizeTables("t2");

                LTrieSettings = new TrieSettings();
                LTrieStorage = new StorageLayer(@"E:\temp\DBreezeTest\DBR1\90000000", LTrieSettings, new DBreezeConfiguration());

                LTrie = new LTrie(LTrieStorage);
                LTrie.Add(((int)2).To_4_bytes_array_BigEndian(), ((int)2).To_4_bytes_array_BigEndian());
                LTrie.Commit();
                LTrie.Dispose();

                var row = tran.Select<int, byte>("t2", 1);

                Console.WriteLine("K: {0}", row.Value);

                tran.RestoreTableFromTheOtherFile("t2", @"E:\temp\DBreezeTest\DBR1\90000000");

                //row = tran.Select<int, byte>("t2", 1);

                Console.WriteLine("K: {0}", row.Value);
            }

            using (var tran = engine.GetTransaction())
            {
                foreach (var row in tran.SelectBackward<int,int>("t2"))
                {
                    Console.WriteLine("Key: {0}", row.Key);
                }
            }
        }
Example #12
0
        /// <summary>
        /// 
        /// </summary>
        private void InitLTrieAscii()
        {
            if (initLTrieAscii)
                return;

            //File.Delete(LianaTrieFileName);

            initLTrieAscii = true;

            LTrieSettings = new TrieSettings();
            LTrieStorage = new StorageLayer(LianaTrieFileName, LTrieSettings, new DBreezeConfiguration());
            //LTrieStorage = new TrieMemoryStorage(50 * 1024 * 1024, LTrieSettings);

            LTrie = new LTrie(LTrieStorage);
        }