protected Repository(string databaseName, ILoggerFactory loggerFactory)
{
path = Environment.GetEnvironmentVariable("DATA_DIR");
logger = loggerFactory.CreateLogger(databaseName);
var cfg = new BTreeDatabaseConfig
{
Creation = CreatePolicy.IF_NEEDED,
CacheSize = new CacheInfo(1, 0, 1),
ErrorFeedback = (prefix, message) =>
{
logger.LogError($"{DateTime.Now} [primary] {prefix}: {message}");
},
ErrorPrefix = databaseName
};
db = BTreeDatabase.Open(Path.Combine(path, databaseName + ".db"), cfg);
//set up secondary database
var scfg = new SecondaryBTreeDatabaseConfig(db, GenerateIndex)
{
Creation = CreatePolicy.IF_NEEDED,
Duplicates = DuplicatesPolicy.SORTED,
ErrorFeedback = (prefix, message) =>
{
logger.LogError($"{DateTime.Now} [secondary] {prefix}: {message}");
}
};
indexDb = SecondaryBTreeDatabase.Open(Path.Combine(path, databaseName + "-index.db"), scfg);
cursor = db.Cursor();
scursor = indexDb.SecondaryCursor();
}
public void RdMfWt()
{
Transaction txn = paramEnv.BeginTransaction();
BTreeCursor dbc = paramDB.Cursor(txn);
try
{
LockingInfo lck = new LockingInfo();
lck.ReadModifyWrite = true;
// Read record.
btCursorFunc(dbc, lck);
// Block the current thread until event is set.
signal.WaitOne();
// Write new records into database.
DatabaseEntry key = new DatabaseEntry(BitConverter.GetBytes(55));
DatabaseEntry data = new DatabaseEntry(BitConverter.GetBytes(55));
dbc.Add(new KeyValuePair <DatabaseEntry, DatabaseEntry>(key, data));
dbc.Close();
txn.Commit();
}
catch (DeadlockException)
{
dbc.Close();
txn.Abort();
}
}
private void GetMultipleDB(string dbFileName,
BTreeDatabaseConfig dbConfig, Transaction txn,
out BTreeDatabase db, out BTreeCursor cursor)
{
if (txn == null)
{
db = BTreeDatabase.Open(dbFileName, dbConfig);
cursor = db.Cursor();
}
else
{
db = BTreeDatabase.Open(
dbFileName, dbConfig, txn);
CursorConfig cursorConfig = new CursorConfig();
cursor = db.Cursor(cursorConfig, txn);
}
KeyValuePair <DatabaseEntry, DatabaseEntry> pair;
DatabaseEntry key, data;
for (int i = 1; i < 100; i++)
{
key = new DatabaseEntry(BitConverter.GetBytes(i));
data = new DatabaseEntry(BitConverter.GetBytes(i));
pair = new KeyValuePair <DatabaseEntry, DatabaseEntry>(key, data);
cursor.Add(pair);
}
if (dbConfig.UseRecordNumbers == true)
{
byte[] bytes = new byte[512];
for (int i = 0; i < 512; i++)
{
bytes[i] = (byte)i;
}
key = new DatabaseEntry(BitConverter.GetBytes(100));
data = new DatabaseEntry(bytes);
pair = new KeyValuePair <DatabaseEntry, DatabaseEntry>(key, data);
cursor.Add(pair);
}
else
{
if (dbConfig.Duplicates == DuplicatesPolicy.UNSORTED ||
dbConfig.Duplicates == DuplicatesPolicy.SORTED)
{
key = new DatabaseEntry(BitConverter.GetBytes(99));
data = new DatabaseEntry(BitConverter.GetBytes(100));
pair = new KeyValuePair <DatabaseEntry, DatabaseEntry>(key, data);
cursor.Add(pair);
}
key = new DatabaseEntry(BitConverter.GetBytes(101));
data = new DatabaseEntry(BitConverter.GetBytes(101));
pair = new KeyValuePair <DatabaseEntry, DatabaseEntry>(key, data);
cursor.Add(pair);
}
}
public void Clear()
{
BTreeCursor cursor = db.Cursor();
while (cursor.MoveNext())
{
db.Delete(cursor.Current.Key);
}
cursor.Close();
}
public void GetCursorInBtreeDBUsingRecno(string home,
string name, out BTreeDatabase db,
out BTreeCursor cursor)
{
string dbFileName = home + "/" + name + ".db";
BTreeDatabaseConfig dbConfig = new BTreeDatabaseConfig();
dbConfig.UseRecordNumbers = true;
dbConfig.Creation = CreatePolicy.IF_NEEDED;
db = BTreeDatabase.Open(dbFileName, dbConfig);
cursor = db.Cursor();
}
public void showAllInventory()
{
BTreeCursor cursor = myDbs.InventoryDB.Cursor();
Inventory theInventory;
while (cursor.MoveNext())
{
theInventory = new Inventory(cursor.Current.Value.Data);
displayInventory(theInventory);
}
cursor.Close();
}
public Ticks ReadBackBerkeleyDBData(BTreeDatabase database, Action <int> updateProgressBar)
{
IEnumerable <ulong> points;
if (m_settings.ReadFromCsv)
{
points = m_indexToPointIDLookup.Skip(1); // First value is always 0 because the timestamp is the first column
}
else
{
points = m_points;
}
if (points == null)
{
ShowMessage("Point list not initialized");
return(new Ticks(0));
}
int count = 0;
ulong value;
long timestamp;
int messageInterval = m_settings.MessageInterval * points.Count();
using (BTreeCursor cursor = database.Cursor())
{
DateTime startTime = DateTime.UtcNow;
while (cursor.MoveNextMultipleKey())
{
using (MultipleKeyDatabaseEntry pairs = cursor.CurrentMultipleKey)
{
foreach (KeyValuePair <DatabaseEntry, DatabaseEntry> p in pairs)
{
timestamp = BitConverter.ToInt64(p.Key.Data, 0);
value = BitConverter.ToUInt64(p.Value.Data, 0);
p.Key.Dispose();
p.Value.Dispose();
count++;
if (count % messageInterval == 0)
{
PercentComplete = (int)((1.0D - (new Ticks(m_endTime.Ticks - timestamp).ToSeconds() / m_timeRange)) * 100.0D);
ShowMessage($"{Environment.NewLine}{count} points read back so far, averaging {(count / (DateTime.UtcNow - startTime).TotalSeconds):N0} points per second.");
updateProgressBar(PercentComplete);
}
}
}
}
return(DateTime.UtcNow - startTime);
}
}
public Dictionary <DatabaseEntry, DatabaseEntry> FetchAll()
{
BTreeCursor cursor = db.Cursor();
Dictionary <DatabaseEntry, DatabaseEntry> dic = new Dictionary <DatabaseEntry, DatabaseEntry>();
while (cursor.MoveNext())
{
dic.Add(cursor.Current.Key, cursor.Current.Value);
}
cursor.Close();
return(dic);
}
public static void GetCursorInBtreeDBWithoutEnv(
string home, string name, out BTreeDatabase db,
out BTreeCursor cursor)
{
string dbFileName = home + "/" + name + ".db";
BTreeDatabaseConfig dbConfig =
new BTreeDatabaseConfig();
dbConfig.Creation = CreatePolicy.IF_NEEDED;
dbConfig.Duplicates = DuplicatesPolicy.UNSORTED;
db = BTreeDatabase.Open(dbFileName, dbConfig);
cursor = db.Cursor();
}
/*
* Move the cursor according to recno. The recno
* starts from 1 and is increased by 1.
*/
public void MoveCursorToRecno(BTreeCursor cursor,
LockingInfo lck)
{
for (uint i = 1; i <= 5; i++)
{
if (lck == null)
{
Assert.IsTrue(cursor.Move(i));
}
else
{
Assert.IsTrue(cursor.Move(i, lck));
}
}
}
public void ConfirmOverwrite()
{
Transaction confirmTxn = paramEnv.BeginTransaction();
BTreeCursor cursor = paramDB.Cursor(confirmTxn);
int i = 0;
Assert.IsTrue(cursor.MoveFirst());
do
{
Assert.AreNotEqual(
cursor.Current.Key.Data,
cursor.Current.Value.Data);
} while (i <= 1000 && cursor.MoveNext());
cursor.Close();
confirmTxn.Commit();
}
public static void GetCursorInBtreeDBInTDS(
string home, string name,
CursorConfig cursorConfig,
out DatabaseEnvironment env, out BTreeDatabase db,
out BTreeCursor cursor, out Transaction txn)
{
string dbFileName = name + ".db";
Configuration.ClearDir(home);
// Open an environment.
DatabaseEnvironmentConfig envConfig =
new DatabaseEnvironmentConfig();
envConfig.Create = true;
envConfig.UseMPool = true;
envConfig.UseTxns = true;
envConfig.NoMMap = false;
envConfig.UseLocking = true;
env = DatabaseEnvironment.Open(home, envConfig);
// Begin a transaction.
txn = env.BeginTransaction();
/*
* Open an btree database. The underlying database
* should be opened with ReadUncommitted if the
* cursor's isolation degree will be set to be 1.
*/
BTreeDatabaseConfig dbConfig = new BTreeDatabaseConfig();
dbConfig.Creation = CreatePolicy.IF_NEEDED;
dbConfig.Env = env;
if (cursorConfig.IsolationDegree == Isolation.DEGREE_ONE)
{
dbConfig.ReadUncommitted = true;
}
db = BTreeDatabase.Open(dbFileName, dbConfig, txn);
// Get a cursor in the transaction.
cursor = db.Cursor(cursorConfig, txn);
}
public void UpdateTxn()
{
int int32Value;
DatabaseEntry data;
// Get a new transaction for updating the db.
TransactionConfig txnConfig =
new TransactionConfig();
txnConfig.IsolationDegree =
Isolation.DEGREE_THREE;
updateTxn =
paramEnv.BeginTransaction(txnConfig);
// Continually putting record to db.
BTreeCursor cursor =
paramDB.Cursor(updateTxn);
// Move the cursor to the first record.
Assert.IsTrue(cursor.MoveFirst());
int i = 0;
try
{
do
{
int32Value = BitConverter.ToInt32(
cursor.Current.Value.Data, 0);
data = new DatabaseEntry(
BitConverter.GetBytes(int32Value - 1));
cursor.Overwrite(data);
} while (i <= 1000 && cursor.MoveNext());
}
catch (DeadlockException)
{
}
finally
{
cursor.Close();
}
}
protected Repository(string databaseName, ILoggerFactory loggerService)
{
logger = loggerService.CreateLogger(databaseName);
path = Environment.GetEnvironmentVariable("DATA_DIR");
var cfg = new BTreeDatabaseConfig
{
Creation = CreatePolicy.IF_NEEDED,
CacheSize = new CacheInfo(1, 0, 1),
ErrorFeedback = (prefix, message) =>
{
logger.LogError($"{prefix}: {message}");
},
ErrorPrefix = databaseName,
Duplicates = DuplicatesPolicy.SORTED,
};
db = BTreeDatabase.Open(Path.Combine(path, databaseName + ".db"), cfg);
cursor = db.Cursor();
}
/*l
* Move the cursor according to a given recno and
* return the current record's recno. The given recno
* and the one got from the cursor should be the same.
*/
public void ReturnRecno(BTreeCursor cursor,
LockingInfo lck)
{
for (uint i = 1; i <= 5; i++)
{
if (lck == null)
{
if (cursor.Move(i) == true)
{
Assert.AreEqual(i, cursor.Recno());
}
}
else
{
if (cursor.Move(i, lck) == true)
{
Assert.AreEqual(i, cursor.Recno(lck));
}
}
}
}
public void ReadTxn()
{
// Get a new transaction for reading the db.
TransactionConfig txnConfig =
new TransactionConfig();
txnConfig.Snapshot = true;
readTxn = paramEnv.BeginTransaction(
txnConfig);
// Get a new cursor for putting record into db.
CursorConfig cursorConfig = new CursorConfig();
cursorConfig.WriteCursor = false;
BTreeCursor cursor = paramDB.Cursor(
cursorConfig, readTxn);
// Continually reading record from db.
try
{
Assert.IsTrue(cursor.MoveFirst());
int i = 0;
do
{
Assert.AreEqual(
BitConverter.ToInt32(
cursor.Current.Key.Data, 0),
BitConverter.ToInt32(
cursor.Current.Value.Data, 0));
} while (i <= 1000 && cursor.MoveNext());
}
catch (DeadlockException)
{
}
finally
{
cursor.Close();
}
}
private void GetMultipleDB(string home, string dbFileName,
BTreeDatabaseConfig dbConfig, out DatabaseEnvironment env,
out Transaction txn, out BTreeDatabase db,
out BTreeCursor cursor)
{
DatabaseEnvironmentConfig envConfig =
new DatabaseEnvironmentConfig();
envConfig.Create = true;
envConfig.UseTxns = true;
envConfig.UseLocking = true;
envConfig.UseLogging = true;
envConfig.UseMPool = true;
env = DatabaseEnvironment.Open(home, envConfig);
txn = env.BeginTransaction();
if (dbConfig == null)
{
dbConfig = new BTreeDatabaseConfig();
}
dbConfig.Env = env;
GetMultipleDB(dbFileName, dbConfig, txn, out db,
out cursor);
}
/// <summary>
/// Creates the temporaray databases for storing the input sets, and then reads the input
/// sets in full and stores them in the databases.
/// </summary>
private void CreateTmpDbs()
{
//
// the temporary databases are BerkeleyDB B+ tree databases. configuration for the
// databases is hardcoded here.
var config = new BTreeDatabaseConfig();
config.Duplicates = DuplicatesPolicy.UNSORTED;
config.CacheSize = new CacheInfo(0, m_planOperator.MemorySize / 2, 4);
config.PageSize = 512;
config.Creation = CreatePolicy.ALWAYS;
m_tmpDbLeftFile = String.Format("~{0}.tmp", Generator.GetRandomFilename(12));
m_tmpDbRightFile = String.Format("~{0}.tmp", Generator.GetRandomFilename(12));
#if DEBUG
m_planOperator.StartIOWork();
#endif
m_tmpDbLeft = BTreeDatabase.Open(m_tmpDbLeftFile, config);
m_tmpDbRight = BTreeDatabase.Open(m_tmpDbRightFile, config);
#if DEBUG
m_planOperator.StopIOWork();
#endif
//
// peek-a-binding from the left and right input streams, to obtain the shared join
// variables between them
var peekLeft = m_left.Peek();
var peekRight = m_right.Peek();
foreach (var b1 in peekLeft.Bindings)
{
var b2 = peekRight[b1.Variable];
if (b2 != null)
{
m_joinVariables.Add(b1.Variable);
}
}
var barrier = new Barrier(2);
//
// do this multi-threaded. because why not?
Action <Operator, BTreeDatabase> dbBuildAction = (op, db) => {
while (op.HasNext)
{
var next = op.Next();
#if DEBUG
lock (m_planOperator) {
m_planOperator.StartCPUWork();
}
#endif
var bKey = new List <Binding>();
foreach (var item in m_joinVariables)
{
bKey.Add(next[item]);
}
#if DEBUG
lock (m_planOperator) {
m_planOperator.StopCPUWork();
m_planOperator.StartIOWork();
}
#endif
var key = new DatabaseEntry(Encoding.DbEncode(Hash(bKey.ToArray())));
var data = new DatabaseEntry(Encoding.DbEncode(next));
db.Put(key, data);
#if DEBUG
lock (m_planOperator) {
m_planOperator.StopIOWork();
}
#endif
}
barrier.SignalAndWait();
};
//
// one goes in a seperate thread...
var t = new Thread(() => dbBuildAction(m_left, m_tmpDbLeft));
t.Start();
//
// ...the other one just in the main thread.
dbBuildAction(m_right, m_tmpDbRight);
//
// create the cursors for moving through the databases
m_cursorLeft = m_tmpDbLeft.Cursor();
m_cursorLeftMovePrev = false;
m_cursorRight = m_tmpDbRight.Cursor();
m_cursorRightMovePrev = false;
m_tmpDbsCreated = true;
}
