public TextDeferredIndexer(DBreezeEngine engine)
{
this.DBreezeEngine = engine;
LTrieSettings = new TrieSettings()
{
InternalTable = true
};
Storage = new StorageLayer(Path.Combine(engine.MainFolder, TableFileName), LTrieSettings, engine.Configuration);
LTrie = new LTrie(Storage);
LTrie.TableName = "DBreeze.TextIndexer";
if (LTrie.Storage.Length > 100000) //Recreating file if its size more then 100KB and it is empty
{
if (LTrie.Count(true) == 0)
{
LTrie.Storage.RecreateFiles();
LTrie.Dispose();
Storage = new StorageLayer(Path.Combine(engine.MainFolder, TableFileName), LTrieSettings, engine.Configuration);
LTrie = new LTrie(Storage);
LTrie.TableName = "DBreeze.TextIndexer";
}
}
if (LTrie.Count(true) > 0)
this.StartDefferedIndexing();
}
public RISR(string fileName, TrieSettings trieSettings, DBreezeConfiguration configuration)
{
this._fileName = fileName;
this._configuration = configuration;
this._trieSettings = trieSettings;
DefaultPointerLen = this._trieSettings.POINTER_LENGTH;
_backupIsActive = this._configuration.Backup.IsActive;
//Transforms fileName into ulong digit
ulFileName = this._configuration.Backup.BackupFNP.ParseFilename(Path.GetFileNameWithoutExtension(this._fileName));
//Setting up RemoteCommander
RIC = new DBreeze.Storage.RemoteInstance.RemoteInstanceCommander(configuration.RICommunicator);
InitFiles();
}
public StorageLayer(string fileName, TrieSettings trieSettings, DBreezeConfiguration configuration)
{
if (trieSettings.StorageWasOverriden)
{
switch (trieSettings.AlternativeTableStorageType)
{
case DBreezeConfiguration.eStorage.DISK:
_tableStorage = (IStorage) new FSR(fileName, trieSettings, configuration);
break;
case DBreezeConfiguration.eStorage.MEMORY:
_tableStorage = (IStorage)new MSR(fileName, trieSettings, configuration);
break;
case DBreezeConfiguration.eStorage.RemoteInstance:
_tableStorage = (IStorage)new RISR(fileName, trieSettings, configuration);
break;
}
}
else
{
switch (configuration.Storage)
{
case DBreezeConfiguration.eStorage.DISK:
_tableStorage = (IStorage)new FSR(fileName, trieSettings, configuration);
break;
case DBreezeConfiguration.eStorage.MEMORY:
_tableStorage = (IStorage)new MSR(fileName, trieSettings, configuration);
break;
case DBreezeConfiguration.eStorage.RemoteInstance:
_tableStorage = (IStorage)new RISR(fileName, trieSettings, configuration);
break;
}
}
}
public RISR(string fileName, TrieSettings trieSettings, DBreezeConfiguration configuration)
{
this._fileName = fileName;
this._configuration = configuration;
this._trieSettings = trieSettings;
DefaultPointerLen = this._trieSettings.POINTER_LENGTH;
_backupIsActive = this._configuration.Backup.IsActive;
//Transforms fileName into ulong digit
ulFileName = this._configuration.Backup.BackupFNP.ParseFilename(Path.GetFileNameWithoutExtension(this._fileName));
//Setting up RemoteCommander
RIC = new DBreeze.Storage.RemoteInstance.RemoteInstanceCommander(configuration.RICommunicator);
InitFiles();
}
/* TODO
* 1. HERE we will add TableNames as RegEx with settings
* 2. Checking Reserverd TableNames prefixes
* 3. User TableName must start from @ut
* 4. GetPhysicalPathToTheUserTable - File with DIrectory Settings for different tables parser (to make reside different tables in different HDDs or even network drives)
*/
private void OpenSchema()
{
LTrieSettings = new TrieSettings()
{
InternalTable = true,
//SkipStorageBuffer = true
};
Storage = new StorageLayer(Path.Combine(Engine.MainFolder, SchemaFileName), LTrieSettings, Engine.Configuration);
LTrie = new LTrie(Storage);
LTrie.TableName = "DBreeze.Scheme";
//Reading lastFileNumber
ReadUserLastFileNumber();
}
/// <summary>
/// Returns table for READ, WRITE FUNC
/// </summary>
/// <param name="userTableName"></param>
/// <returns></returns>
internal LTrie GetTable(string userTableName)
{
string tableName = GetUserTableNameAsString(userTableName);
//TODO pattern based mapping If table doesn't exist we create it with properties which could be supplied after db init as regex theme.
//Schema protocol: 2 bytes - protocol version, other data
//For protocol 1: first 8 bytes will be TheFileName, starting from db10000-dbN (0-N ulong). up to 10000 are reserved for dbreeze.
//Table names are UTF-8 based, no limits
ulong fileName = 0;
OpenTable otl = null;
_sync_openTablesHolder.EnterUpgradeableReadLock();
try
{
_openTablesHolder.TryGetValue(tableName, out otl);
if (otl != null)
{
//Try to increase usage and return LTrie
otl.Add();
return otl.Trie;
}
//Probably table Exists in db but not in openTablesHolder
_sync_openTablesHolder.EnterWriteLock();
try
{
//UpgradeableRead recheck
_openTablesHolder.TryGetValue(tableName, out otl);
if (otl != null)
{
//Try to increase usage and return LTrie
otl.Add();
return otl.Trie;
}
byte[] btTableName = GetUserTableNameAsByte(userTableName);
//Trying to get fileName from cache
fileName = this.cachedTableNames.GetFileName(tableName);
// LTrieRow row = null;
bool tableExists = false;
if (fileName == 0)
{
LTrieRow row = LTrie.GetKey(btTableName, false);
if (row.Exists)
{
tableExists = true;
byte[] fullValue = row.GetFullValue(false);
//Can be parsed different. First protocol version is 1
ushort schemeProtocol = fullValue.Substring(0, 2).To_UInt16_BigEndian();
switch (schemeProtocol)
{
case 1:
fileName = fullValue.Substring(2, 8).To_UInt64_BigEndian();
break;
default:
throw DBreezeException.Throw(DBreezeException.eDBreezeExceptions.SCHEME_FILE_PROTOCOL_IS_UNKNOWN);
}
}
else
{
tableExists = false;
//Creating new table.
//Checking table name validity
//this will throw exception, if not valid
DbUserTables.UserTableNameIsOk(userTableName);
//Creating such table and renewing LastFileNumber counter
//Adding to LastFileNumber
LastFileNumber++;
////Deleting physical files related to the table, if they existed - normally they should not
//DeleteAllReleatedTableFiles(Path.Combine(Engine.MainFolder, LastFileNumber.ToString()));
byte[] lft = LastFileNumber.To_8_bytes_array_BigEndian();
//Writing this number to Schema file
LTrie.Add(Encoding.UTF8.GetBytes(LastFileNumberKeyName), lft);
//Creating table self and writing to Schema file
LTrie.Add(btTableName,
new byte[] { 0, 1 } //Protocol version 1
.Concat(lft)); //Number of the file
//Committing both records
LTrie.Commit();
fileName = LastFileNumber;
this.cachedTableNames.Add(tableName, fileName);
}
}
else
tableExists = true;
//Creating LTrie, adding it to _openTablesHolder
//Seeting up Trie TableName, OTHER SETTINGS
TrieSettings ts = new TrieSettings();
IStorage storage = null;
////Checking if default Flusg Disk behaviour was overriden
//ts.DiskFlushBehaviour = Engine.Configuration.DiskFlushBehaviour;
////Checking if we have alternative DiskFlush behaviour
//foreach (var pattern in Engine.Configuration.AlternativeDiskFlushBehaviour)
//{
// //pattern.Key
// if (DbUserTables.PatternsIntersect(pattern.Key, userTableName))
// {
// ts.DiskFlushBehaviour = pattern.Value;
// break;
// }
//}
string alternativeTableLocation = String.Empty;
if (CheckAlternativeTableLocationsIntersections(userTableName, out alternativeTableLocation))
{
ts.StorageWasOverriden = true;
if (alternativeTableLocation == String.Empty)
{
ts.AlternativeTableStorageType = DBreezeConfiguration.eStorage.MEMORY;
storage = new StorageLayer(Path.Combine(Engine.MainFolder, fileName.ToString()), ts, Engine.Configuration);
}
else
{
ts.AlternativeTableStorageType = DBreezeConfiguration.eStorage.DISK;
ts.AlternativeTableStorageFolder = alternativeTableLocation;
DirectoryInfo diAlt = new DirectoryInfo(alternativeTableLocation);
if (!diAlt.Exists)
diAlt.Create();
if (!tableExists)
{
//Deleting physical files related to the table, if they existed - normally they should not
DeleteAllReleatedTableFiles(Path.Combine(ts.AlternativeTableStorageFolder, LastFileNumber.ToString()));
}
storage = new StorageLayer(Path.Combine(ts.AlternativeTableStorageFolder, fileName.ToString()), ts, Engine.Configuration);
}
}
else
{
if (!tableExists)
{
//Deleting physical files related to the table, if they existed - normally they should not
DeleteAllReleatedTableFiles(Path.Combine(Engine.MainFolder, LastFileNumber.ToString()));
}
storage = new StorageLayer(Path.Combine(Engine.MainFolder, fileName.ToString()), ts, Engine.Configuration);
}
//storage = new StorageLayer(Path.Combine(Engine.MainFolder, fileName.ToString()), ts, Engine.Configuration);
LTrie trie = new LTrie(storage);
//Setting LTrie user table name
trie.TableName = userTableName;
//_openTablesHolder.Add(tableName, trie);
//Automatically increased usage in OpenTable constructor
_openTablesHolder.Add(tableName, new OpenTable(trie));
return trie;
}
catch (System.Exception ex)
{
//CASCADE
throw ex;
}
finally
{
_sync_openTablesHolder.ExitWriteLock();
}
}
catch (Exception ex)
{
throw DBreezeException.Throw(DBreezeException.eDBreezeExceptions.SCHEME_GET_TABLE_WRITE_FAILED, tableName, ex);
}
finally
{
_sync_openTablesHolder.ExitUpgradeableReadLock();
}
}
public NestedTableStorage(IStorage masterStorage, TrieSettings trieSettings)
{
_masterStorage = masterStorage;
_trieSettings = trieSettings;
}
private void RestoreNotFinishedTransactions()
{
//TODO Trie settings from the table must be taken from schema (when they will differ)
//STORE FILE NAME of rollback not table name
try
{
byte[] btCommittedTablesNames =null;
List<string> committedTablesNames = new List<string>();
if (LTrie.Count(false) == 0) //All ok
{
LTrie.RemoveAll(true);
return;
}
string physicalPathToTheUserTable = String.Empty;
//Settigns and storage for Committed tables !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! MUST BE TAKEN FROM SCHEMA, FOR NOW DEFAULT
TrieSettings ltrSet = null;
IStorage storage = null;
DBreeze.LianaTrie.LTrie ltrie = null;
foreach (var row in LTrie.IterateForward())
{
btCommittedTablesNames = row.GetFullValue(true);
committedTablesNames = System.Text.Encoding.UTF8.GetString(btCommittedTablesNames).DeserializeXml<List<string>>();
foreach (var fn in committedTablesNames)
{
//Trying to get path from the Schema, there is universal function for getting table physical TABLE FULL PATH /NAME
physicalPathToTheUserTable = Engine.DBreezeSchema.GetPhysicalPathToTheUserTable(fn);
//Returned path can be empty, if no more such table
if (physicalPathToTheUserTable == String.Empty)
continue;
//We don't restore in-memory tables
if (physicalPathToTheUserTable == "MEMORY")
continue;
//we open ltrie, and it automatically restores rollback
ltrSet = new TrieSettings(); //!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! MUST BE TAKEN FROM SCHEMA, FOR NOW DEFAULT
//storage = new TrieDiskStorage(physicalPathToTheUserTable, ltrSet, Engine.Configuration);
storage = new StorageLayer(physicalPathToTheUserTable, ltrSet, Engine.Configuration);
ltrie = new LTrie(storage);
//closing trie, that Schema could open it again
ltrie.Dispose();
////Deleting rollback file for such table
//physicalPathToTheUserTable += ".rol";
//System.IO.File.Delete(physicalPathToTheUserTable);
}
committedTablesNames.Clear();
}
//If all ok, recreate file
LTrie.RemoveAll(true);
}
catch (OperationCanceledException ex)
{
throw ex;
}
//catch (System.Threading.ThreadAbortException ex)
//{
// //We don'T make DBisOperable = false;
// throw ex;
//}
catch (Exception ex)
{
//BRINGS TO DB NOT OPERATABLE
this.Engine.DBisOperable = false;
this.Engine.DBisOperableReason = "TransactionsCoordinator.RestoreNotFinishedTransaction";
//NOT CASCADE ADD EXCEPTION
throw DBreezeException.Throw(DBreezeException.eDBreezeExceptions.CLEAN_ROLLBACK_FILES_FOR_FINISHED_TRANSACTIONS_FAILED);
}
}
private void Init()
{
try
{
LTrieSettings = new TrieSettings()
{
InternalTable = true,
//SkipStorageBuffer = true
};
Storage = new StorageLayer(Path.Combine(Engine.MainFolder, JournalFileName), LTrieSettings, Engine.Configuration);
//Storage = new TrieDiskStorage(Path.Combine(Engine.MainFolder, JournalFileName), LTrieSettings, Engine.Configuration);
LTrie = new LTrie(Storage);
LTrie.TableName = "DBreeze.TranJournal";
this.RestoreNotFinishedTransactions();
}
catch (Exception ex)
{
//CASCADE
throw ex;
}
}
public FSR(string fileName, TrieSettings trieSettings, DBreezeConfiguration configuration)
{
this._fileName = fileName;
this._configuration = configuration;
this._trieSettings = trieSettings;
DefaultPointerLen = this._trieSettings.POINTER_LENGTH;
_backupIsActive = this._configuration.Backup.IsActive;
//Transforms fileName into ulong digit
ulFileName = this._configuration.Backup.BackupFNP.ParseFilename(Path.GetFileNameWithoutExtension(this._fileName));
InitFiles();
}
public MSR(string fileName, TrieSettings trieSettings,DBreezeConfiguration configuration)
{
this._fileName = fileName;
this._configuration = configuration;
this._trieSettings = trieSettings;
DefaultPointerLen = this._trieSettings.POINTER_LENGTH;
InitFiles();
}
private void TestKrome()
{
using (var tran = engine.GetTransaction())
{
byte[] ptr = tran.InsertDataBlock("t1", null, new byte[] { 1, 2, 3 });
tran.Insert<int, byte[]>("t1", 1, ptr);
//NestedTable nt = tran.InsertTable<int>("t1", 1, 0);
//nt.Insert<int, int>(1, 1);
tran.Commit();
}
using (var tran = engine.GetTransaction())
{
var row = tran.Select<int,byte[]>("t1",1);
byte[] val = row.Value;
//NestedTable nt = tran.SelectTable<int>("t1", 1, 0);
//var row = nt.Select<int, int>(1);
tran.RemoveAllKeys("t1", true);
byte[] res = tran.SelectDataBlock("t1", val);
//Console.WriteLine("Key: {0}", row.Value);
}
return;
//using (var tran = engine.GetTransaction())
//{
// for (int i = 0; i < 1000000; i++)
// {
// tran.Insert<int, byte>("t1", i, 1);
// }
// tran.Commit();
//}
//Console.WriteLine("***");
//byte bt = 1;
//using (var tran = engine.GetTransaction())
//{
// DBreeze.Diagnostic.SpeedStatistic.StartCounter("a");
// foreach (var row in tran.SelectForward<int, byte>("t1"))
// {
// bt = row.Value;
// // Console.WriteLine("Key: {0}", row.Key);
// }
// DBreeze.Diagnostic.SpeedStatistic.PrintOut("a",true);
//}
//return;
using (var tran = engine.GetTransaction())
{
tran.Insert<int, byte>("t1", 1, 1);
tran.Insert<int, byte>("t2", 1, 1);
tran.Insert<int, byte>("t3", 1, 1);
tran.Commit();
}
using (var tran = engine.GetTransaction())
{
tran.SynchronizeTables("t2");
LTrieSettings = new TrieSettings();
LTrieStorage = new StorageLayer(@"E:\temp\DBreezeTest\DBR1\90000000", LTrieSettings, new DBreezeConfiguration());
LTrie = new LTrie(LTrieStorage);
LTrie.Add(((int)2).To_4_bytes_array_BigEndian(), ((int)2).To_4_bytes_array_BigEndian());
LTrie.Commit();
LTrie.Dispose();
var row = tran.Select<int, byte>("t2", 1);
Console.WriteLine("K: {0}", row.Value);
tran.RestoreTableFromTheOtherFile("t2", @"E:\temp\DBreezeTest\DBR1\90000000");
//row = tran.Select<int, byte>("t2", 1);
Console.WriteLine("K: {0}", row.Value);
}
using (var tran = engine.GetTransaction())
{
foreach (var row in tran.SelectBackward<int,int>("t2"))
{
Console.WriteLine("Key: {0}", row.Key);
}
}
}
/// <summary>
///
/// </summary>
private void InitLTrieAscii()
{
if (initLTrieAscii)
return;
//File.Delete(LianaTrieFileName);
initLTrieAscii = true;
LTrieSettings = new TrieSettings();
LTrieStorage = new StorageLayer(LianaTrieFileName, LTrieSettings, new DBreezeConfiguration());
//LTrieStorage = new TrieMemoryStorage(50 * 1024 * 1024, LTrieSettings);
LTrie = new LTrie(LTrieStorage);
}