public TextDeferredIndexer(DBreezeEngine engine) { this.DBreezeEngine = engine; LTrieSettings = new TrieSettings() { InternalTable = true }; Storage = new StorageLayer(Path.Combine(engine.MainFolder, TableFileName), LTrieSettings, engine.Configuration); LTrie = new LTrie(Storage); LTrie.TableName = "DBreeze.TextIndexer"; if (LTrie.Storage.Length > 100000) //Recreating file if its size more then 100KB and it is empty { if (LTrie.Count(true) == 0) { LTrie.Storage.RecreateFiles(); LTrie.Dispose(); Storage = new StorageLayer(Path.Combine(engine.MainFolder, TableFileName), LTrieSettings, engine.Configuration); LTrie = new LTrie(Storage); LTrie.TableName = "DBreeze.TextIndexer"; } } if (LTrie.Count(true) > 0) this.StartDefferedIndexing(); }
//public LTrieWriteCache(IStorage storage, bool overWriteIsAllowed) public LTrieWriteCache(LTrie trie) { Trie = trie; DefaultPointerLen = Trie.Storage.TrieSettings.POINTER_LENGTH; ////Represents pointer to the root of DefaultPointerLen size //rootPointerAsByte = ((ulong)Trie.Storage.TrieSettings.ROOT_START).To_8_bytes_array_BigEndian().Substring(8 - DefaultPointerLen, DefaultPointerLen); IsNestedTable = Trie.Storage.TrieSettings.IsNestedTable; }
public LTrieRootNode(LTrie tree) { Tree = tree; DefaultPointerLen = Tree.Storage.TrieSettings.POINTER_LENGHT; DefaultRootSize = Tree.Storage.TrieSettings.ROOT_SIZE; //me = new byte[Tree.Storage.TreeSettings.ROOT_SIZE]; LinkToZeroNode = new byte[DefaultPointerLen]; this.EmptyPointer = new byte[DefaultPointerLen]; //Reading Root Node this.ReadRootNode(); }
private void RestoreNotFinishedTransactions() { //TODO Trie settings from the table must be taken from schema (when they will differ) //STORE FILE NAME of rollback not table name try { byte[] btCommittedTablesNames = null; List <string> committedTablesNames = new List <string>(); if (LTrie.Count(false) == 0) //All ok { LTrie.RemoveAll(true); return; } string physicalPathToTheUserTable = String.Empty; //Settigns and storage for Committed tables !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! MUST BE TAKEN FROM SCHEMA, FOR NOW DEFAULT TrieSettings ltrSet = null; IStorage storage = null; DBreeze.LianaTrie.LTrie ltrie = null; foreach (var row in LTrie.IterateForward()) { btCommittedTablesNames = row.GetFullValue(true); committedTablesNames = System.Text.Encoding.UTF8.GetString(btCommittedTablesNames).DeserializeXml <List <string> >(); foreach (var fn in committedTablesNames) { //Trying to get path from the Schema, there is universal function for getting table physical TABLE FULL PATH /NAME physicalPathToTheUserTable = Engine.DBreezeSchema.GetPhysicalPathToTheUserTable(fn); //Returned path can be empty, if no more such table if (physicalPathToTheUserTable == String.Empty) { continue; } //We don't restore in-memory tables if (physicalPathToTheUserTable == "MEMORY") { continue; } //we open ltrie, and it automatically restores rollback ltrSet = new TrieSettings(); //!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! MUST BE TAKEN FROM SCHEMA, FOR NOW DEFAULT //storage = new TrieDiskStorage(physicalPathToTheUserTable, ltrSet, Engine.Configuration); storage = new StorageLayer(physicalPathToTheUserTable, ltrSet, Engine.Configuration); ltrie = new LTrie(storage); //closing trie, that Schema could open it again ltrie.Dispose(); ////Deleting rollback file for such table //physicalPathToTheUserTable += ".rol"; //System.IO.File.Delete(physicalPathToTheUserTable); } committedTablesNames.Clear(); } //If all ok, recreate file LTrie.RemoveAll(true); } catch (OperationCanceledException ex) { throw ex; } //catch (System.Threading.ThreadAbortException ex) //{ // //We don'T make DBisOperable = false; // throw ex; //} catch (Exception ex) { //BRINGS TO DB NOT OPERATABLE this.Engine.DBisOperable = false; this.Engine.DBisOperableReason = "TransactionsCoordinator.RestoreNotFinishedTransaction"; //NOT CASCADE ADD EXCEPTION throw DBreezeException.Throw(DBreezeException.eDBreezeExceptions.CLEAN_ROLLBACK_FILES_FOR_FINISHED_TRANSACTIONS_FAILED); } }
/* TODO * 1. HERE we will add TableNames as RegEx with settings * 2. Checking Reserverd TableNames prefixes * 3. User TableName must start from @ut * 4. GetPhysicalPathToTheUserTable - File with DIrectory Settings for different tables parser (to make reside different tables in different HDDs or even network drives) */ private void OpenSchema() { LTrieSettings = new TrieSettings() { InternalTable = true, //SkipStorageBuffer = true }; Storage = new StorageLayer(Path.Combine(Engine.MainFolder, SchemaFileName), LTrieSettings, Engine.Configuration); LTrie = new LTrie(Storage); LTrie.TableName = "DBreeze.Scheme"; //Reading lastFileNumber ReadUserLastFileNumber(); }
/// <summary> /// Returns table for READ, WRITE FUNC /// </summary> /// <param name="userTableName"></param> /// <returns></returns> internal LTrie GetTable(string userTableName) { string tableName = GetUserTableNameAsString(userTableName); //TODO pattern based mapping If table doesn't exist we create it with properties which could be supplied after db init as regex theme. //Schema protocol: 2 bytes - protocol version, other data //For protocol 1: first 8 bytes will be TheFileName, starting from db10000-dbN (0-N ulong). up to 10000 are reserved for dbreeze. //Table names are UTF-8 based, no limits ulong fileName = 0; OpenTable otl = null; _sync_openTablesHolder.EnterUpgradeableReadLock(); try { _openTablesHolder.TryGetValue(tableName, out otl); if (otl != null) { //Try to increase usage and return LTrie otl.Add(); return otl.Trie; } //Probably table Exists in db but not in openTablesHolder _sync_openTablesHolder.EnterWriteLock(); try { //UpgradeableRead recheck _openTablesHolder.TryGetValue(tableName, out otl); if (otl != null) { //Try to increase usage and return LTrie otl.Add(); return otl.Trie; } byte[] btTableName = GetUserTableNameAsByte(userTableName); //Trying to get fileName from cache fileName = this.cachedTableNames.GetFileName(tableName); // LTrieRow row = null; bool tableExists = false; if (fileName == 0) { LTrieRow row = LTrie.GetKey(btTableName, false); if (row.Exists) { tableExists = true; byte[] fullValue = row.GetFullValue(false); //Can be parsed different. First protocol version is 1 ushort schemeProtocol = fullValue.Substring(0, 2).To_UInt16_BigEndian(); switch (schemeProtocol) { case 1: fileName = fullValue.Substring(2, 8).To_UInt64_BigEndian(); break; default: throw DBreezeException.Throw(DBreezeException.eDBreezeExceptions.SCHEME_FILE_PROTOCOL_IS_UNKNOWN); } } else { tableExists = false; //Creating new table. //Checking table name validity //this will throw exception, if not valid DbUserTables.UserTableNameIsOk(userTableName); //Creating such table and renewing LastFileNumber counter //Adding to LastFileNumber LastFileNumber++; ////Deleting physical files related to the table, if they existed - normally they should not //DeleteAllReleatedTableFiles(Path.Combine(Engine.MainFolder, LastFileNumber.ToString())); byte[] lft = LastFileNumber.To_8_bytes_array_BigEndian(); //Writing this number to Schema file LTrie.Add(Encoding.UTF8.GetBytes(LastFileNumberKeyName), lft); //Creating table self and writing to Schema file LTrie.Add(btTableName, new byte[] { 0, 1 } //Protocol version 1 .Concat(lft)); //Number of the file //Committing both records LTrie.Commit(); fileName = LastFileNumber; this.cachedTableNames.Add(tableName, fileName); } } else tableExists = true; //Creating LTrie, adding it to _openTablesHolder //Seeting up Trie TableName, OTHER SETTINGS TrieSettings ts = new TrieSettings(); IStorage storage = null; ////Checking if default Flusg Disk behaviour was overriden //ts.DiskFlushBehaviour = Engine.Configuration.DiskFlushBehaviour; ////Checking if we have alternative DiskFlush behaviour //foreach (var pattern in Engine.Configuration.AlternativeDiskFlushBehaviour) //{ // //pattern.Key // if (DbUserTables.PatternsIntersect(pattern.Key, userTableName)) // { // ts.DiskFlushBehaviour = pattern.Value; // break; // } //} string alternativeTableLocation = String.Empty; if (CheckAlternativeTableLocationsIntersections(userTableName, out alternativeTableLocation)) { ts.StorageWasOverriden = true; if (alternativeTableLocation == String.Empty) { ts.AlternativeTableStorageType = DBreezeConfiguration.eStorage.MEMORY; storage = new StorageLayer(Path.Combine(Engine.MainFolder, fileName.ToString()), ts, Engine.Configuration); } else { ts.AlternativeTableStorageType = DBreezeConfiguration.eStorage.DISK; ts.AlternativeTableStorageFolder = alternativeTableLocation; DirectoryInfo diAlt = new DirectoryInfo(alternativeTableLocation); if (!diAlt.Exists) diAlt.Create(); if (!tableExists) { //Deleting physical files related to the table, if they existed - normally they should not DeleteAllReleatedTableFiles(Path.Combine(ts.AlternativeTableStorageFolder, LastFileNumber.ToString())); } storage = new StorageLayer(Path.Combine(ts.AlternativeTableStorageFolder, fileName.ToString()), ts, Engine.Configuration); } } else { if (!tableExists) { //Deleting physical files related to the table, if they existed - normally they should not DeleteAllReleatedTableFiles(Path.Combine(Engine.MainFolder, LastFileNumber.ToString())); } storage = new StorageLayer(Path.Combine(Engine.MainFolder, fileName.ToString()), ts, Engine.Configuration); } //storage = new StorageLayer(Path.Combine(Engine.MainFolder, fileName.ToString()), ts, Engine.Configuration); LTrie trie = new LTrie(storage); //Setting LTrie user table name trie.TableName = userTableName; //_openTablesHolder.Add(tableName, trie); //Automatically increased usage in OpenTable constructor _openTablesHolder.Add(tableName, new OpenTable(trie)); return trie; } catch (System.Exception ex) { //CASCADE throw ex; } finally { _sync_openTablesHolder.ExitWriteLock(); } } catch (Exception ex) { throw DBreezeException.Throw(DBreezeException.eDBreezeExceptions.SCHEME_GET_TABLE_WRITE_FAILED, tableName, ex); } finally { _sync_openTablesHolder.ExitUpgradeableReadLock(); } }
private void RestoreNotFinishedTransactions() { //TODO Trie settings from the table must be taken from schema (when they will differ) //STORE FILE NAME of rollback not table name try { byte[] btCommittedTablesNames =null; List<string> committedTablesNames = new List<string>(); if (LTrie.Count(false) == 0) //All ok { LTrie.RemoveAll(true); return; } string physicalPathToTheUserTable = String.Empty; //Settigns and storage for Committed tables !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! MUST BE TAKEN FROM SCHEMA, FOR NOW DEFAULT TrieSettings ltrSet = null; IStorage storage = null; DBreeze.LianaTrie.LTrie ltrie = null; foreach (var row in LTrie.IterateForward()) { btCommittedTablesNames = row.GetFullValue(true); committedTablesNames = System.Text.Encoding.UTF8.GetString(btCommittedTablesNames).DeserializeXml<List<string>>(); foreach (var fn in committedTablesNames) { //Trying to get path from the Schema, there is universal function for getting table physical TABLE FULL PATH /NAME physicalPathToTheUserTable = Engine.DBreezeSchema.GetPhysicalPathToTheUserTable(fn); //Returned path can be empty, if no more such table if (physicalPathToTheUserTable == String.Empty) continue; //We don't restore in-memory tables if (physicalPathToTheUserTable == "MEMORY") continue; //we open ltrie, and it automatically restores rollback ltrSet = new TrieSettings(); //!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! MUST BE TAKEN FROM SCHEMA, FOR NOW DEFAULT //storage = new TrieDiskStorage(physicalPathToTheUserTable, ltrSet, Engine.Configuration); storage = new StorageLayer(physicalPathToTheUserTable, ltrSet, Engine.Configuration); ltrie = new LTrie(storage); //closing trie, that Schema could open it again ltrie.Dispose(); ////Deleting rollback file for such table //physicalPathToTheUserTable += ".rol"; //System.IO.File.Delete(physicalPathToTheUserTable); } committedTablesNames.Clear(); } //If all ok, recreate file LTrie.RemoveAll(true); } catch (OperationCanceledException ex) { throw ex; } //catch (System.Threading.ThreadAbortException ex) //{ // //We don'T make DBisOperable = false; // throw ex; //} catch (Exception ex) { //BRINGS TO DB NOT OPERATABLE this.Engine.DBisOperable = false; this.Engine.DBisOperableReason = "TransactionsCoordinator.RestoreNotFinishedTransaction"; //NOT CASCADE ADD EXCEPTION throw DBreezeException.Throw(DBreezeException.eDBreezeExceptions.CLEAN_ROLLBACK_FILES_FOR_FINISHED_TRANSACTIONS_FAILED); } }
private void Init() { try { LTrieSettings = new TrieSettings() { InternalTable = true, //SkipStorageBuffer = true }; Storage = new StorageLayer(Path.Combine(Engine.MainFolder, JournalFileName), LTrieSettings, Engine.Configuration); //Storage = new TrieDiskStorage(Path.Combine(Engine.MainFolder, JournalFileName), LTrieSettings, Engine.Configuration); LTrie = new LTrie(Storage); LTrie.TableName = "DBreeze.TranJournal"; this.RestoreNotFinishedTransactions(); } catch (Exception ex) { //CASCADE throw ex; } }
public void FinishTransaction(ulong tranNumber) { //_sync_transactionsTables.EnterReadLock(); _sync_transactionsTables.EnterWriteLock(); try { Dictionary<string, ITransactable> tbls = null; _transactionsTables.TryGetValue(tranNumber, out tbls); if (tbls != null) { //Starting procedure //1. Saving all table names into db - needed in case if something happens (Power loss or whatever). // Then restarted TransactionalJournal will delete rollback files for these tables (they are all committed) List<string> committedTablesNames = new List<string>(); foreach (var tt in tbls) { committedTablesNames.Add(tt.Key); } string serTbls = committedTablesNames.SerializeXml(); byte[] btSerTbls = System.Text.Encoding.UTF8.GetBytes(serTbls); byte[] key = tranNumber.To_8_bytes_array_BigEndian(); LTrie.Add(ref key, ref btSerTbls); LTrie.Commit(); //2. Calling transaction End for all tables try { foreach (var tt in tbls) { tt.Value.ITRCommitFinished(); } } catch (Exception ex) { //CASCADE from ITRCommitFinished, brings to NON-OPERATABLE throw ex; } //3. Deleting Record in Journal LTrie.Remove(ref key); LTrie.Commit(); //Clearing transaction number tbls.Clear(); _transactionsTables.Remove(tranNumber); //When Transaction File becomes big we try to clean it. if (LTrie.Storage.Length > MaxlengthOfTransactionFile && _transactionsTables.Count() == 0) { LTrie.Storage.RecreateFiles(); LTrie.Dispose(); Storage = new StorageLayer(Path.Combine(Engine.MainFolder, JournalFileName), LTrieSettings, Engine.Configuration); LTrie = new LTrie(Storage); LTrie.TableName = "DBreeze.TranJournal"; } } } catch (System.Exception ex) { //CASCADE throw ex; } finally { _sync_transactionsTables.ExitWriteLock(); //_sync_transactionsTables.ExitReadLock(); } }
//object lock_nestedTblAccess = new object(); /// <summary> /// /// </summary> /// <param name="row"></param> /// <param name="tableIndex"></param> /// <param name="masterTrie"></param> /// <param name="insertTable">Regulates if InsertTable or SelectTable was called (ability to create table if it doesn't exist)</param> /// <param name="useCache">Regulates READ table thread or WRITE table thread - visibilityscope</param> /// <returns></returns> public NestedTable GetTable(LTrieRow row, ref byte[] btKey, uint tableIndex, LTrie masterTrie, bool insertTable, bool useCache) { //Console.WriteLine(useCache.ToString() + " " + System.Threading.Thread.CurrentThread.ManagedThreadId.ToString()); try { if (!insertTable && !row.Exists) { //For Readers, which couldn't allocate table, we return empty DbInTable return new NestedTable(null, false, false); } if (insertTable) { //settign up modification thread this.NestedTablesCoordinator.ModificationThreadId = System.Threading.Thread.CurrentThread.ManagedThreadId; } if (masterTrie == null) masterTrie = this; NestedTableInternal dit = null; //this one we will return byte[] fullValuePointer = null; if (insertTable) TableIsModified = true; //flag will permit commit and rollback byte[] val = null; long Id_RootStart = 0; if (row.Exists) { val = row.GetPartialValue(tableIndex * this.Storage.TrieSettings.ROOT_SIZE, this.Storage.TrieSettings.ROOT_SIZE, useCache); if (val == null || val.Length < this.Storage.TrieSettings.ROOT_SIZE) { //For Readers, which couldn't allocate table, we return empty DbInTable if (!insertTable) { return new NestedTable(null, false, false); } //Here can appear only WRITERS, which couldn't allocate table byte[] btValue = new byte[this.Storage.TrieSettings.ROOT_SIZE]; long valueStartPointer = 0; fullValuePointer = this.AddPartially(ref btKey, ref btValue, tableIndex * this.Storage.TrieSettings.ROOT_SIZE, out valueStartPointer); Id_RootStart = valueStartPointer + (tableIndex * this.Storage.TrieSettings.ROOT_SIZE); lock (this.NestedTablesCoordinator.lock_nestedTblAccess) { dit = new DataTypes.NestedTableInternal(true, masterTrie, Id_RootStart, (tableIndex * this.Storage.TrieSettings.ROOT_SIZE), false,this,ref btKey); this.NestedTablesCoordinator.AddNestedTable(ref btKey, fullValuePointer.DynamicLength_To_UInt64_BigEndian(), Id_RootStart, dit); //if (!insertTable) //{ //If we open the table for read, we increase read value dit.quantityOpenReads++; //} } } else { //then we must open/create DbInTable Id_RootStart = row.ValueStartPointer + (tableIndex * this.Storage.TrieSettings.ROOT_SIZE); fullValuePointer = row.LinkToValue; lock (this.NestedTablesCoordinator.lock_nestedTblAccess) { dit = this.NestedTablesCoordinator.GetTable(ref btKey, Id_RootStart); if (dit == null) { dit = new DataTypes.NestedTableInternal(true, masterTrie, Id_RootStart, (tableIndex * this.Storage.TrieSettings.ROOT_SIZE), useCache, this, ref btKey); this.NestedTablesCoordinator.AddNestedTable(ref btKey, fullValuePointer.DynamicLength_To_UInt64_BigEndian(), Id_RootStart, dit); } //if (!insertTable) //{ //If we open the table for read, we increase read value dit.quantityOpenReads++; //} } } } else { //Every new row with DbInTable starts from here //Here can appear only WRITERS, which couldn't allocate table //creating space for the root byte[] btValue = new byte[this.Storage.TrieSettings.ROOT_SIZE]; //adding empty root to the place long valueStartPointer = 0; fullValuePointer = this.AddPartially(ref btKey, ref btValue, tableIndex * this.Storage.TrieSettings.ROOT_SIZE, out valueStartPointer); Id_RootStart = valueStartPointer + (tableIndex * this.Storage.TrieSettings.ROOT_SIZE); lock (this.NestedTablesCoordinator.lock_nestedTblAccess) { dit = new DataTypes.NestedTableInternal(true, masterTrie, Id_RootStart, (tableIndex * this.Storage.TrieSettings.ROOT_SIZE), false, this, ref btKey); this.NestedTablesCoordinator.AddNestedTable(ref btKey, fullValuePointer.DynamicLength_To_UInt64_BigEndian(), Id_RootStart, dit); //if (!insertTable) //{ //If we open the table for read, we increase read value dit.quantityOpenReads++; //} } } //Console.WriteLine("Creating NestedTable " + btKey.ToBytesString("")); //this.NestedTablesCoordinator.AddNestedTable(ref btKey, fullValuePointer.DynamicLength_To_UInt64_BigEndian(), Id_RootStart, dit); return new NestedTable(dit, insertTable, true); } catch (Exception ex) { this.RollBack(); //Cascade throw ex; } }
//public OpenTable() //{ // Add(); //} public OpenTable(LTrie trie) { this.Trie = trie; Add(); }
private void TestKrome() { using (var tran = engine.GetTransaction()) { byte[] ptr = tran.InsertDataBlock("t1", null, new byte[] { 1, 2, 3 }); tran.Insert<int, byte[]>("t1", 1, ptr); //NestedTable nt = tran.InsertTable<int>("t1", 1, 0); //nt.Insert<int, int>(1, 1); tran.Commit(); } using (var tran = engine.GetTransaction()) { var row = tran.Select<int,byte[]>("t1",1); byte[] val = row.Value; //NestedTable nt = tran.SelectTable<int>("t1", 1, 0); //var row = nt.Select<int, int>(1); tran.RemoveAllKeys("t1", true); byte[] res = tran.SelectDataBlock("t1", val); //Console.WriteLine("Key: {0}", row.Value); } return; //using (var tran = engine.GetTransaction()) //{ // for (int i = 0; i < 1000000; i++) // { // tran.Insert<int, byte>("t1", i, 1); // } // tran.Commit(); //} //Console.WriteLine("***"); //byte bt = 1; //using (var tran = engine.GetTransaction()) //{ // DBreeze.Diagnostic.SpeedStatistic.StartCounter("a"); // foreach (var row in tran.SelectForward<int, byte>("t1")) // { // bt = row.Value; // // Console.WriteLine("Key: {0}", row.Key); // } // DBreeze.Diagnostic.SpeedStatistic.PrintOut("a",true); //} //return; using (var tran = engine.GetTransaction()) { tran.Insert<int, byte>("t1", 1, 1); tran.Insert<int, byte>("t2", 1, 1); tran.Insert<int, byte>("t3", 1, 1); tran.Commit(); } using (var tran = engine.GetTransaction()) { tran.SynchronizeTables("t2"); LTrieSettings = new TrieSettings(); LTrieStorage = new StorageLayer(@"E:\temp\DBreezeTest\DBR1\90000000", LTrieSettings, new DBreezeConfiguration()); LTrie = new LTrie(LTrieStorage); LTrie.Add(((int)2).To_4_bytes_array_BigEndian(), ((int)2).To_4_bytes_array_BigEndian()); LTrie.Commit(); LTrie.Dispose(); var row = tran.Select<int, byte>("t2", 1); Console.WriteLine("K: {0}", row.Value); tran.RestoreTableFromTheOtherFile("t2", @"E:\temp\DBreezeTest\DBR1\90000000"); //row = tran.Select<int, byte>("t2", 1); Console.WriteLine("K: {0}", row.Value); } using (var tran = engine.GetTransaction()) { foreach (var row in tran.SelectBackward<int,int>("t2")) { Console.WriteLine("Key: {0}", row.Key); } } }
/// <summary> /// /// </summary> private void InitLTrieAscii() { if (initLTrieAscii) return; //File.Delete(LianaTrieFileName); initLTrieAscii = true; LTrieSettings = new TrieSettings(); LTrieStorage = new StorageLayer(LianaTrieFileName, LTrieSettings, new DBreezeConfiguration()); //LTrieStorage = new TrieMemoryStorage(50 * 1024 * 1024, LTrieSettings); LTrie = new LTrie(LTrieStorage); }