public void PutRecordCase1(HashDatabase db, Transaction txn)
{
byte[] bigArray = new byte[262144];
for (int i = 0; i < 50; i++)
{
if (txn == null)
{
db.Put(new DatabaseEntry(BitConverter.GetBytes(i)),
new DatabaseEntry(BitConverter.GetBytes(i)));
}
else
{
db.Put(new DatabaseEntry(BitConverter.GetBytes(i)),
new DatabaseEntry(BitConverter.GetBytes(i)), txn);
}
}
for (int i = 50; i < 100; i++)
{
if (txn == null)
{
db.Put(new DatabaseEntry(bigArray),
new DatabaseEntry(bigArray));
}
else
{
db.Put(new DatabaseEntry(bigArray),
new DatabaseEntry(bigArray), txn);
}
}
}
/*
* Insert a document to the database.
*/
public void InsertToDatabase(Document document)
{
DatabaseEntry key = new DatabaseEntry(BitConverter.GetBytes(document.Id));
DatabaseEntry value = new DatabaseEntry(document.GetByteArray());
hashDatabase.Put(key, value);
key = new DatabaseEntry(Encoding.UTF8.GetBytes(document.Url));
value = new DatabaseEntry(BitConverter.GetBytes(document.Id));
urlDatabase.Put(key, value);
}
protected void AddToDb(string keyval, byte[] dataval)
{
var key = new DatabaseEntry(Encoding.UTF8.GetBytes(keyval));
var data = new DatabaseEntry(dataval);
db.Put(key, data);
}
/// <summary>
/// Inserts the specified (integer, string) pair of internal- and external representations
/// into the dictionary.
/// </summary>
/// <param name="internalValue">The internal representation.</param>
/// <param name="externalValue">The external representation.</param>
private void Insert(long internalValue, string externalValue)
{
//
// the (external, internal) key-value pair is put in the string2integer dictionary
var str2LongKey = new DatabaseEntry(Util.Encoding.DbEncode(externalValue));
var str2LongValue = new DatabaseEntry(Util.Encoding.DbEncode(internalValue));
m_dbStr2Long.Put(str2LongKey, str2LongValue);
//
// similarly, the (internal, external) key-value pair goes into the integer2string
// dictionary
var long2StrKey = new DatabaseEntry(Util.Encoding.DbEncode(internalValue));
var long2StrValue = new DatabaseEntry(Util.Encoding.DbEncode(externalValue));
m_dbLong2Str.Put(long2StrKey, long2StrValue);
}
public void TestCompactWithoutTxn()
{
int i, nRecs;
nRecs = 10000;
testName = "TestCompactWithoutTxn";
SetUpTest(true);
string hashDBFileName = testHome + "/" + testName + ".db";
HashDatabaseConfig hashDBConfig = new HashDatabaseConfig();
hashDBConfig.Creation = CreatePolicy.ALWAYS;
// The minimum page size
hashDBConfig.PageSize = 512;
hashDBConfig.HashComparison =
new EntryComparisonDelegate(dbIntCompare);
using (HashDatabase hashDB = HashDatabase.Open(
hashDBFileName, hashDBConfig)) {
DatabaseEntry key;
DatabaseEntry data;
// Fill the database with entries from 0 to 9999
for (i = 0; i < nRecs; i++)
{
key = new DatabaseEntry(BitConverter.GetBytes(i));
data = new DatabaseEntry(BitConverter.GetBytes(i));
hashDB.Put(key, data);
}
/*
* Delete entries below 500, between 3000 and
* 5000 and above 7000
*/
for (i = 0; i < nRecs; i++)
{
if (i < 500 || i > 7000 || (i < 5000 && i > 3000))
{
key = new DatabaseEntry(BitConverter.GetBytes(i));
hashDB.Delete(key);
}
}
hashDB.Sync();
long fileSize = new FileInfo(hashDBFileName).Length;
// Compact database
CompactConfig cCfg = new CompactConfig();
cCfg.FillPercentage = 30;
cCfg.Pages = 10;
cCfg.Timeout = 1000;
cCfg.TruncatePages = true;
cCfg.start = new DatabaseEntry(BitConverter.GetBytes(1));
cCfg.stop = new DatabaseEntry(BitConverter.GetBytes(7000));
CompactData compactData = hashDB.Compact(cCfg);
Assert.IsFalse((compactData.Deadlocks == 0) &&
(compactData.Levels == 0) &&
(compactData.PagesExamined == 0) &&
(compactData.PagesFreed == 0) &&
(compactData.PagesTruncated == 0));
hashDB.Sync();
long compactedFileSize = new FileInfo(hashDBFileName).Length;
Assert.Less(compactedFileSize, fileSize);
}
}
/// <summary>
/// Builds the index over the buckets that belong to this index.
/// </summary>
internal void Build()
{
var currentAtom = Int64.MaxValue;
var start = new long[3];
var count = new long[3];
var cursors = new TripleCursor[3];
cursors[0] = m_sBucket.OpenRead();
cursors[1] = m_pBucket.OpenRead();
cursors[2] = m_oBucket.OpenRead();
//
// the index building algorithm is fairly straightforward: all three buckets are opened
// simultaneously, and we go through them such that we are always dealing with the same
// atom X for all three buckets. at any time we are at (or before) X in the S-bucket
// where X is in the S-position, at X in the P-bucket where it is in the P-position,
// and at X in the O-bucket where it is in the O-position. when we are done with X we
// move on to the next atom and never see a relevant X value again. in this way, we
// only need one pass over all three buckets to build the index.
long peek;
while (cursors[0].HasNext || cursors[1].HasNext || cursors[2].HasNext)
{
//
// find the next smallest atom value to handle
if (cursors[0].HasNext)
{
peek = cursors[0].Peek().S.InternalValue;
if (peek < currentAtom)
{
currentAtom = peek;
}
}
if (cursors[1].HasNext)
{
peek = cursors[1].Peek().P.InternalValue;
if (peek < currentAtom)
{
currentAtom = peek;
}
}
if (cursors[2].HasNext)
{
peek = cursors[2].Peek().O.InternalValue;
if (peek < currentAtom)
{
currentAtom = peek;
}
}
//
// see how much atoms with the selected value there are with that value in the S
// position
start[0] = -1;
count[0] = -1;
if (cursors[0].HasNext && cursors[0].Peek().S.InternalValue == currentAtom)
{
start[0] = cursors[0].Position;
count[0] = 0;
do
{
count[0]++;
cursors[0].Next();
} while (cursors[0].HasNext && cursors[0].Peek().S.InternalValue == currentAtom);
}
//
// see how much atoms with the selected value there are with that value in the P
// position
start[1] = -1;
count[1] = -1;
if (cursors[1].HasNext && cursors[1].Peek().P.InternalValue == currentAtom)
{
start[1] = cursors[1].Position;
count[1] = 0;
do
{
count[1]++;
cursors[1].Next();
} while (cursors[1].HasNext && cursors[1].Peek().P.InternalValue == currentAtom);
}
//
// see how much atoms with the selected value there are with that value in the O
// position
start[2] = -1;
count[2] = -1;
if (cursors[2].HasNext && cursors[2].Peek().O.InternalValue == currentAtom)
{
start[2] = cursors[2].Position;
count[2] = 0;
do
{
count[2]++;
cursors[2].Next();
} while (cursors[2].HasNext && cursors[2].Peek().O.InternalValue == currentAtom);
}
//
// create and insert the index payload for the atom with the current value into the
// index database
var payload = new byte[8 * 6];
var i = 0;
for (int j = 0; j < start.Length; j++)
{
Encoding.DbEncode(start[j]).CopyTo(payload, i);
i += 8;
Encoding.DbEncode(count[j]).CopyTo(payload, i);
i += 8;
}
var key = new DatabaseEntry(Encoding.DbEncode(currentAtom));
var value = new DatabaseEntry(payload);
m_db.Put(key, value);
currentAtom = Int64.MaxValue;
}
}
/*
* Test the external file database with or without environment.
* 1. Config and open the environment;
* 2. Verify the environment external file configs;
* 3. Config and open the database;
* 4. Verify the database external file configs;
* 5. Insert and verify some external file data by database methods;
* 6. Insert some external file data by cursor, update it and verify
* the update by database stream and cursor;
* 7. Verify the stats;
* 8. Close all handles.
* If "blobdbt" is true, set the data DatabaseEntry.ExternalFile as
* true, otherwise make the data DatabaseEntry reach the external file
* threshold in size.
*/
void TestBlobHashDatabase(uint env_threshold, string env_blobdir,
uint db_threshold, string db_blobdir, bool blobdbt)
{
if (env_threshold == 0 && db_threshold == 0)
{
return;
}
string hashDBName =
testHome + "/" + testName + ".db";
Configuration.ClearDir(testHome);
HashDatabaseConfig cfg = new HashDatabaseConfig();
cfg.Creation = CreatePolicy.ALWAYS;
string blrootdir = "__db_bl";
// Open the environment and verify the external file config.
if (env_threshold > 0)
{
DatabaseEnvironmentConfig envConfig =
new DatabaseEnvironmentConfig();
envConfig.AutoCommit = true;
envConfig.Create = true;
envConfig.UseMPool = true;
envConfig.UseLogging = true;
envConfig.UseTxns = true;
envConfig.UseLocking = true;
envConfig.ExternalFileThreshold = env_threshold;
if (env_blobdir != null)
{
envConfig.ExternalFileDir = env_blobdir;
blrootdir = env_blobdir;
}
DatabaseEnvironment env = DatabaseEnvironment.Open(
testHome, envConfig);
if (env_blobdir == null)
{
Assert.IsNull(env.ExternalFileDir);
}
else
{
Assert.AreEqual(0,
env.ExternalFileDir.CompareTo(env_blobdir));
}
Assert.AreEqual(env_threshold,
env.ExternalFileThreshold);
cfg.Env = env;
hashDBName = testName + ".db";
}
// Open the database and verify the external file config.
if (db_threshold > 0)
{
cfg.ExternalFileThreshold = db_threshold;
}
if (db_blobdir != null)
{
cfg.ExternalFileDir = db_blobdir;
/*
* The external file directory setting in the database
* is effective only when it is opened without
* an environment.
*/
if (cfg.Env == null)
{
blrootdir = db_blobdir;
}
}
HashDatabase db = HashDatabase.Open(hashDBName, cfg);
Assert.AreEqual(
db_threshold > 0 ? db_threshold : env_threshold,
db.ExternalFileThreshold);
if (db_blobdir == null && cfg.Env == null)
{
Assert.IsNull(db.ExternalFileDir);
}
else
{
Assert.AreEqual(0,
db.ExternalFileDir.CompareTo(blrootdir));
}
// Insert and verify some external file data by database
// methods.
string[] records = { "a", "b", "c", "d", "e", "f", "g", "h",
"i", "j", "k", "l", "m", "n", "o", "p","q","r", "s",
"t", "u", "v", "w", "x", "y", "z" };
DatabaseEntry kdbt = new DatabaseEntry();
DatabaseEntry ddbt = new DatabaseEntry();
byte[] kdata, ddata;
string str;
KeyValuePair <DatabaseEntry, DatabaseEntry> pair;
ddbt.ExternalFile = blobdbt;
Assert.AreEqual(blobdbt, ddbt.ExternalFile);
for (int i = 0; i < records.Length; i++)
{
kdata = BitConverter.GetBytes(i);
str = records[i];
if (!blobdbt)
{
for (int j = 0; j < db_threshold; j++)
{
str = str + records[i];
}
}
ddata = Encoding.ASCII.GetBytes(str);
kdbt.Data = kdata;
ddbt.Data = ddata;
db.Put(kdbt, ddbt);
try
{
pair = db.Get(kdbt);
}
catch (DatabaseException)
{
db.Close();
if (cfg.Env != null)
{
cfg.Env.Close();
}
throw new TestException();
}
Assert.AreEqual(ddata, pair.Value.Data);
}
/*
* Insert some external file data by cursor, update it and
* verify the update by database stream.
*/
kdata = BitConverter.GetBytes(records.Length);
ddata = Encoding.ASCII.GetBytes("abc");
kdbt.Data = kdata;
ddbt.Data = ddata;
ddbt.ExternalFile = true;
Assert.IsTrue(ddbt.ExternalFile);
pair =
new KeyValuePair <DatabaseEntry, DatabaseEntry>(kdbt, ddbt);
CursorConfig dbcConfig = new CursorConfig();
Transaction txn = null;
if (cfg.Env != null)
{
txn = cfg.Env.BeginTransaction();
}
HashCursor cursor = db.Cursor(dbcConfig, txn);
cursor.Add(pair);
DatabaseStreamConfig dbsc = new DatabaseStreamConfig();
dbsc.SyncPerWrite = true;
DatabaseStream dbs = cursor.DbStream(dbsc);
Assert.AreNotEqual(null, dbs);
Assert.IsFalse(dbs.GetConfig.ReadOnly);
Assert.IsTrue(dbs.GetConfig.SyncPerWrite);
Assert.AreEqual(3, dbs.Size());
DatabaseEntry sdbt = dbs.Read(0, 3);
Assert.IsNotNull(sdbt);
Assert.AreEqual(ddata, sdbt.Data);
sdbt = new DatabaseEntry(Encoding.ASCII.GetBytes("defg"));
Assert.IsTrue(dbs.Write(sdbt, 3));
Assert.AreEqual(7, dbs.Size());
sdbt = dbs.Read(0, 7);
Assert.IsNotNull(sdbt);
Assert.AreEqual(Encoding.ASCII.GetBytes("abcdefg"), sdbt.Data);
dbs.Close();
/*
* Verify the database stream can not write when it is
* configured to be read-only.
*/
dbsc.ReadOnly = true;
dbs = cursor.DbStream(dbsc);
Assert.IsTrue(dbs.GetConfig.ReadOnly);
try
{
Assert.IsFalse(dbs.Write(sdbt, 7));
throw new TestException();
}
catch (DatabaseException)
{
}
dbs.Close();
// Verify the update by cursor.
Assert.IsTrue(cursor.Move(kdbt, true));
pair = cursor.Current;
Assert.AreEqual(Encoding.ASCII.GetBytes("abcdefg"),
pair.Value.Data);
cursor.Close();
if (cfg.Env != null)
{
txn.Commit();
}
/*
* Verify the external files are created in the expected
* location.
* This part of test is disabled since BTreeDatabase.BlobSubDir
* is not exposed to users.
*/
//if (cfg.Env != null)
// blrootdir = testHome + "/" + blrootdir;
//string blobdir = blrootdir + "/" + db.BlobSubDir;
//Assert.AreEqual(records.Length + 1,
// Directory.GetFiles(blobdir, "__db.bl*").Length);
//Assert.AreEqual(1,
// Directory.GetFiles(blobdir, "__db_blob_meta.db").Length);
// Verify the stats.
HashStats st = db.Stats();
Assert.AreEqual(records.Length + 1, st.nExternalFiles);
// Close all handles.
db.Close();
if (cfg.Env != null)
{
cfg.Env.Close();
}
/*
* Remove the default external file directory
* when it is not under the test home.
*/
if (db_blobdir == null && cfg.Env == null)
{
Directory.Delete("__db_bl", true);
}
}