public TextDeferredIndexer(DBreezeEngine engine) { this.DBreezeEngine = engine; LTrieSettings = new TrieSettings() { InternalTable = true }; Storage = new StorageLayer(Path.Combine(engine.MainFolder, TableFileName), LTrieSettings, engine.Configuration); LTrie = new LTrie(Storage); LTrie.TableName = "DBreeze.TextIndexer"; if (LTrie.Storage.Length > 100000) //Recreating file if its size more then 100KB and it is empty { if (LTrie.Count(true) == 0) { LTrie.Storage.RecreateFiles(); LTrie.Dispose(); Storage = new StorageLayer(Path.Combine(engine.MainFolder, TableFileName), LTrieSettings, engine.Configuration); LTrie = new LTrie(Storage); LTrie.TableName = "DBreeze.TextIndexer"; } } if (LTrie.Count(true) > 0) this.StartDefferedIndexing(); }
public RISR(string fileName, TrieSettings trieSettings, DBreezeConfiguration configuration) { this._fileName = fileName; this._configuration = configuration; this._trieSettings = trieSettings; DefaultPointerLen = this._trieSettings.POINTER_LENGTH; _backupIsActive = this._configuration.Backup.IsActive; //Transforms fileName into ulong digit ulFileName = this._configuration.Backup.BackupFNP.ParseFilename(Path.GetFileNameWithoutExtension(this._fileName)); //Setting up RemoteCommander RIC = new DBreeze.Storage.RemoteInstance.RemoteInstanceCommander(configuration.RICommunicator); InitFiles(); }
public StorageLayer(string fileName, TrieSettings trieSettings, DBreezeConfiguration configuration) { if (trieSettings.StorageWasOverriden) { switch (trieSettings.AlternativeTableStorageType) { case DBreezeConfiguration.eStorage.DISK: _tableStorage = (IStorage) new FSR(fileName, trieSettings, configuration); break; case DBreezeConfiguration.eStorage.MEMORY: _tableStorage = (IStorage)new MSR(fileName, trieSettings, configuration); break; case DBreezeConfiguration.eStorage.RemoteInstance: _tableStorage = (IStorage)new RISR(fileName, trieSettings, configuration); break; } } else { switch (configuration.Storage) { case DBreezeConfiguration.eStorage.DISK: _tableStorage = (IStorage)new FSR(fileName, trieSettings, configuration); break; case DBreezeConfiguration.eStorage.MEMORY: _tableStorage = (IStorage)new MSR(fileName, trieSettings, configuration); break; case DBreezeConfiguration.eStorage.RemoteInstance: _tableStorage = (IStorage)new RISR(fileName, trieSettings, configuration); break; } } }
/* TODO * 1. HERE we will add TableNames as RegEx with settings * 2. Checking Reserverd TableNames prefixes * 3. User TableName must start from @ut * 4. GetPhysicalPathToTheUserTable - File with DIrectory Settings for different tables parser (to make reside different tables in different HDDs or even network drives) */ private void OpenSchema() { LTrieSettings = new TrieSettings() { InternalTable = true, //SkipStorageBuffer = true }; Storage = new StorageLayer(Path.Combine(Engine.MainFolder, SchemaFileName), LTrieSettings, Engine.Configuration); LTrie = new LTrie(Storage); LTrie.TableName = "DBreeze.Scheme"; //Reading lastFileNumber ReadUserLastFileNumber(); }
/// <summary> /// Returns table for READ, WRITE FUNC /// </summary> /// <param name="userTableName"></param> /// <returns></returns> internal LTrie GetTable(string userTableName) { string tableName = GetUserTableNameAsString(userTableName); //TODO pattern based mapping If table doesn't exist we create it with properties which could be supplied after db init as regex theme. //Schema protocol: 2 bytes - protocol version, other data //For protocol 1: first 8 bytes will be TheFileName, starting from db10000-dbN (0-N ulong). up to 10000 are reserved for dbreeze. //Table names are UTF-8 based, no limits ulong fileName = 0; OpenTable otl = null; _sync_openTablesHolder.EnterUpgradeableReadLock(); try { _openTablesHolder.TryGetValue(tableName, out otl); if (otl != null) { //Try to increase usage and return LTrie otl.Add(); return otl.Trie; } //Probably table Exists in db but not in openTablesHolder _sync_openTablesHolder.EnterWriteLock(); try { //UpgradeableRead recheck _openTablesHolder.TryGetValue(tableName, out otl); if (otl != null) { //Try to increase usage and return LTrie otl.Add(); return otl.Trie; } byte[] btTableName = GetUserTableNameAsByte(userTableName); //Trying to get fileName from cache fileName = this.cachedTableNames.GetFileName(tableName); // LTrieRow row = null; bool tableExists = false; if (fileName == 0) { LTrieRow row = LTrie.GetKey(btTableName, false); if (row.Exists) { tableExists = true; byte[] fullValue = row.GetFullValue(false); //Can be parsed different. First protocol version is 1 ushort schemeProtocol = fullValue.Substring(0, 2).To_UInt16_BigEndian(); switch (schemeProtocol) { case 1: fileName = fullValue.Substring(2, 8).To_UInt64_BigEndian(); break; default: throw DBreezeException.Throw(DBreezeException.eDBreezeExceptions.SCHEME_FILE_PROTOCOL_IS_UNKNOWN); } } else { tableExists = false; //Creating new table. //Checking table name validity //this will throw exception, if not valid DbUserTables.UserTableNameIsOk(userTableName); //Creating such table and renewing LastFileNumber counter //Adding to LastFileNumber LastFileNumber++; ////Deleting physical files related to the table, if they existed - normally they should not //DeleteAllReleatedTableFiles(Path.Combine(Engine.MainFolder, LastFileNumber.ToString())); byte[] lft = LastFileNumber.To_8_bytes_array_BigEndian(); //Writing this number to Schema file LTrie.Add(Encoding.UTF8.GetBytes(LastFileNumberKeyName), lft); //Creating table self and writing to Schema file LTrie.Add(btTableName, new byte[] { 0, 1 } //Protocol version 1 .Concat(lft)); //Number of the file //Committing both records LTrie.Commit(); fileName = LastFileNumber; this.cachedTableNames.Add(tableName, fileName); } } else tableExists = true; //Creating LTrie, adding it to _openTablesHolder //Seeting up Trie TableName, OTHER SETTINGS TrieSettings ts = new TrieSettings(); IStorage storage = null; ////Checking if default Flusg Disk behaviour was overriden //ts.DiskFlushBehaviour = Engine.Configuration.DiskFlushBehaviour; ////Checking if we have alternative DiskFlush behaviour //foreach (var pattern in Engine.Configuration.AlternativeDiskFlushBehaviour) //{ // //pattern.Key // if (DbUserTables.PatternsIntersect(pattern.Key, userTableName)) // { // ts.DiskFlushBehaviour = pattern.Value; // break; // } //} string alternativeTableLocation = String.Empty; if (CheckAlternativeTableLocationsIntersections(userTableName, out alternativeTableLocation)) { ts.StorageWasOverriden = true; if (alternativeTableLocation == String.Empty) { ts.AlternativeTableStorageType = DBreezeConfiguration.eStorage.MEMORY; storage = new StorageLayer(Path.Combine(Engine.MainFolder, fileName.ToString()), ts, Engine.Configuration); } else { ts.AlternativeTableStorageType = DBreezeConfiguration.eStorage.DISK; ts.AlternativeTableStorageFolder = alternativeTableLocation; DirectoryInfo diAlt = new DirectoryInfo(alternativeTableLocation); if (!diAlt.Exists) diAlt.Create(); if (!tableExists) { //Deleting physical files related to the table, if they existed - normally they should not DeleteAllReleatedTableFiles(Path.Combine(ts.AlternativeTableStorageFolder, LastFileNumber.ToString())); } storage = new StorageLayer(Path.Combine(ts.AlternativeTableStorageFolder, fileName.ToString()), ts, Engine.Configuration); } } else { if (!tableExists) { //Deleting physical files related to the table, if they existed - normally they should not DeleteAllReleatedTableFiles(Path.Combine(Engine.MainFolder, LastFileNumber.ToString())); } storage = new StorageLayer(Path.Combine(Engine.MainFolder, fileName.ToString()), ts, Engine.Configuration); } //storage = new StorageLayer(Path.Combine(Engine.MainFolder, fileName.ToString()), ts, Engine.Configuration); LTrie trie = new LTrie(storage); //Setting LTrie user table name trie.TableName = userTableName; //_openTablesHolder.Add(tableName, trie); //Automatically increased usage in OpenTable constructor _openTablesHolder.Add(tableName, new OpenTable(trie)); return trie; } catch (System.Exception ex) { //CASCADE throw ex; } finally { _sync_openTablesHolder.ExitWriteLock(); } } catch (Exception ex) { throw DBreezeException.Throw(DBreezeException.eDBreezeExceptions.SCHEME_GET_TABLE_WRITE_FAILED, tableName, ex); } finally { _sync_openTablesHolder.ExitUpgradeableReadLock(); } }
public NestedTableStorage(IStorage masterStorage, TrieSettings trieSettings) { _masterStorage = masterStorage; _trieSettings = trieSettings; }
private void RestoreNotFinishedTransactions() { //TODO Trie settings from the table must be taken from schema (when they will differ) //STORE FILE NAME of rollback not table name try { byte[] btCommittedTablesNames =null; List<string> committedTablesNames = new List<string>(); if (LTrie.Count(false) == 0) //All ok { LTrie.RemoveAll(true); return; } string physicalPathToTheUserTable = String.Empty; //Settigns and storage for Committed tables !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! MUST BE TAKEN FROM SCHEMA, FOR NOW DEFAULT TrieSettings ltrSet = null; IStorage storage = null; DBreeze.LianaTrie.LTrie ltrie = null; foreach (var row in LTrie.IterateForward()) { btCommittedTablesNames = row.GetFullValue(true); committedTablesNames = System.Text.Encoding.UTF8.GetString(btCommittedTablesNames).DeserializeXml<List<string>>(); foreach (var fn in committedTablesNames) { //Trying to get path from the Schema, there is universal function for getting table physical TABLE FULL PATH /NAME physicalPathToTheUserTable = Engine.DBreezeSchema.GetPhysicalPathToTheUserTable(fn); //Returned path can be empty, if no more such table if (physicalPathToTheUserTable == String.Empty) continue; //We don't restore in-memory tables if (physicalPathToTheUserTable == "MEMORY") continue; //we open ltrie, and it automatically restores rollback ltrSet = new TrieSettings(); //!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! MUST BE TAKEN FROM SCHEMA, FOR NOW DEFAULT //storage = new TrieDiskStorage(physicalPathToTheUserTable, ltrSet, Engine.Configuration); storage = new StorageLayer(physicalPathToTheUserTable, ltrSet, Engine.Configuration); ltrie = new LTrie(storage); //closing trie, that Schema could open it again ltrie.Dispose(); ////Deleting rollback file for such table //physicalPathToTheUserTable += ".rol"; //System.IO.File.Delete(physicalPathToTheUserTable); } committedTablesNames.Clear(); } //If all ok, recreate file LTrie.RemoveAll(true); } catch (OperationCanceledException ex) { throw ex; } //catch (System.Threading.ThreadAbortException ex) //{ // //We don'T make DBisOperable = false; // throw ex; //} catch (Exception ex) { //BRINGS TO DB NOT OPERATABLE this.Engine.DBisOperable = false; this.Engine.DBisOperableReason = "TransactionsCoordinator.RestoreNotFinishedTransaction"; //NOT CASCADE ADD EXCEPTION throw DBreezeException.Throw(DBreezeException.eDBreezeExceptions.CLEAN_ROLLBACK_FILES_FOR_FINISHED_TRANSACTIONS_FAILED); } }
private void Init() { try { LTrieSettings = new TrieSettings() { InternalTable = true, //SkipStorageBuffer = true }; Storage = new StorageLayer(Path.Combine(Engine.MainFolder, JournalFileName), LTrieSettings, Engine.Configuration); //Storage = new TrieDiskStorage(Path.Combine(Engine.MainFolder, JournalFileName), LTrieSettings, Engine.Configuration); LTrie = new LTrie(Storage); LTrie.TableName = "DBreeze.TranJournal"; this.RestoreNotFinishedTransactions(); } catch (Exception ex) { //CASCADE throw ex; } }
public FSR(string fileName, TrieSettings trieSettings, DBreezeConfiguration configuration) { this._fileName = fileName; this._configuration = configuration; this._trieSettings = trieSettings; DefaultPointerLen = this._trieSettings.POINTER_LENGTH; _backupIsActive = this._configuration.Backup.IsActive; //Transforms fileName into ulong digit ulFileName = this._configuration.Backup.BackupFNP.ParseFilename(Path.GetFileNameWithoutExtension(this._fileName)); InitFiles(); }
public MSR(string fileName, TrieSettings trieSettings,DBreezeConfiguration configuration) { this._fileName = fileName; this._configuration = configuration; this._trieSettings = trieSettings; DefaultPointerLen = this._trieSettings.POINTER_LENGTH; InitFiles(); }
private void TestKrome() { using (var tran = engine.GetTransaction()) { byte[] ptr = tran.InsertDataBlock("t1", null, new byte[] { 1, 2, 3 }); tran.Insert<int, byte[]>("t1", 1, ptr); //NestedTable nt = tran.InsertTable<int>("t1", 1, 0); //nt.Insert<int, int>(1, 1); tran.Commit(); } using (var tran = engine.GetTransaction()) { var row = tran.Select<int,byte[]>("t1",1); byte[] val = row.Value; //NestedTable nt = tran.SelectTable<int>("t1", 1, 0); //var row = nt.Select<int, int>(1); tran.RemoveAllKeys("t1", true); byte[] res = tran.SelectDataBlock("t1", val); //Console.WriteLine("Key: {0}", row.Value); } return; //using (var tran = engine.GetTransaction()) //{ // for (int i = 0; i < 1000000; i++) // { // tran.Insert<int, byte>("t1", i, 1); // } // tran.Commit(); //} //Console.WriteLine("***"); //byte bt = 1; //using (var tran = engine.GetTransaction()) //{ // DBreeze.Diagnostic.SpeedStatistic.StartCounter("a"); // foreach (var row in tran.SelectForward<int, byte>("t1")) // { // bt = row.Value; // // Console.WriteLine("Key: {0}", row.Key); // } // DBreeze.Diagnostic.SpeedStatistic.PrintOut("a",true); //} //return; using (var tran = engine.GetTransaction()) { tran.Insert<int, byte>("t1", 1, 1); tran.Insert<int, byte>("t2", 1, 1); tran.Insert<int, byte>("t3", 1, 1); tran.Commit(); } using (var tran = engine.GetTransaction()) { tran.SynchronizeTables("t2"); LTrieSettings = new TrieSettings(); LTrieStorage = new StorageLayer(@"E:\temp\DBreezeTest\DBR1\90000000", LTrieSettings, new DBreezeConfiguration()); LTrie = new LTrie(LTrieStorage); LTrie.Add(((int)2).To_4_bytes_array_BigEndian(), ((int)2).To_4_bytes_array_BigEndian()); LTrie.Commit(); LTrie.Dispose(); var row = tran.Select<int, byte>("t2", 1); Console.WriteLine("K: {0}", row.Value); tran.RestoreTableFromTheOtherFile("t2", @"E:\temp\DBreezeTest\DBR1\90000000"); //row = tran.Select<int, byte>("t2", 1); Console.WriteLine("K: {0}", row.Value); } using (var tran = engine.GetTransaction()) { foreach (var row in tran.SelectBackward<int,int>("t2")) { Console.WriteLine("Key: {0}", row.Key); } } }
/// <summary> /// /// </summary> private void InitLTrieAscii() { if (initLTrieAscii) return; //File.Delete(LianaTrieFileName); initLTrieAscii = true; LTrieSettings = new TrieSettings(); LTrieStorage = new StorageLayer(LianaTrieFileName, LTrieSettings, new DBreezeConfiguration()); //LTrieStorage = new TrieMemoryStorage(50 * 1024 * 1024, LTrieSettings); LTrie = new LTrie(LTrieStorage); }