private static void CreateSimplesKeyValue(LMDBEnvironment env, int count)
{
Logger.LogInfo("CreateSimplesKeyValue");
DateTime dtStart = DateTime.Now;
LMDBTransaction tx = env.BeginTransaction(TransactionBeginFlags.NoSync);
using (var db = tx.OpenDatabase(null,
new DatabaseConfiguration {
Flags = DatabaseOpenFlags.Create
}))
{
Logger.LogInfo("count:" + count);
for (int i = 0; i < count; i++)
{
string key = String.Format("key_{0}", i);
string value = String.Format("value_{0}", i);
tx.Put(db, key, value);
}
tx.Commit();
}
DateTime dtStop = DateTime.Now;
TimeSpan ts = dtStop - dtStart;
string str = String.Format("Time elapsed for set:{0} ms", ts.TotalMilliseconds);
Logger.LogInfo(str);
}
public LmdbArchive(string databasePath)
{
var databaseDir = Path.GetDirectoryName(databasePath);
this.lmdbEnvFactory = () => LMDBEnvironment.Create(databaseDir);
this.databaseName = Path.GetFileName(databasePath);
}
public LmdbHelperTests()
{
_directory = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString());
_env = LMDBEnvironment.Create(_directory, disableAsync: true);
_env.MapSize = 100 * 1024 * 1024;
_env.Open();
_writer = new StreamWriter(new FileStream($"trace_{_correlationId}.log", FileMode.OpenOrCreate))
{
AutoFlush = true
};
if (OutputTraceLog)
{
TheTrace.Tracer = (level, s, os) =>
{
lock (_lock)
{
var message = $"{DateTime.Now.ToString("yyyy-MM-dd:HH-mm-ss.fff")}\t{_correlationId}\t{level}\t{(os.Length == 0 ? s : string.Format(s, os))}";
try
{
_writer.WriteLine(message);
}
catch
{
}
}
};
}
}
static void Main(string[] args)
{
Logger.LogInfo("Main...");
int count = 100000;
if (args.Length == 1)
{
count = Convert.ToInt32(args[0]);
}
string str = String.Format("count={0}", count);
Logger.LogInfo(str);
using (LMDBEnvironment env = CreateEnv())
{
CreateAnEntry(env);
CreateSimplesKeyValue(env, count);
GetSimplesKeyValue(env, count);
EnumData(env);
Logger.LogInfo("auto env.Dispose...");
//env.Dispose();
}
Logger.LogInfo("exit...");
}
private static void GetSimplesKeyValue(LMDBEnvironment env, int count)
{
Logger.LogInfo("GetSimplesKeyValue");
DateTime dtStart = DateTime.Now;
LMDBTransaction tx = env.BeginTransaction(TransactionBeginFlags.ReadOnly);
using (LMDBDatabase db = tx.OpenDatabase(null))
{
Logger.LogInfo("count:" + count);
for (int i = 0; i < count; i++)
{
string key = String.Format("key_{0}", i);
//string value = String.Format("value_{0}", i);
var value = tx.Get(db, key);
if (i % 10000 == 0)
{
string strD = String.Format("key:{0} => value:{1}", key, value);
Logger.LogInfo(strD);
}
}
}
DateTime dtStop = DateTime.Now;
TimeSpan ts = dtStop - dtStart;
string str = String.Format("Time elapsed for get:{0} ms", ts.TotalMilliseconds);
Logger.LogInfo(str);
}
private static void StoreFile(string path)
{
string key = path;
string value = String.Empty;
byte[] buffer = File.ReadAllBytes(path);
LMDBEnvironment _env;
string dir = "c:\\temp\\cache_net10B";
_env = new LMDBEnvironment(dir);
_env.MaxDatabases = 2;
_env.MapSize = 10485760 * 100;
_env.Open();
DateTime dtStart = DateTime.Now;
var tx = _env.BeginTransaction();
var db = tx.OpenDatabase("DB", new DatabaseConfiguration {
Flags = DatabaseOpenFlags.Create
});
var enc = System.Text.Encoding.UTF8;
tx.Put(db, enc.GetBytes(key), buffer);
tx.Commit();
db.Dispose();
DateTime dtStop = DateTime.Now;
TimeSpan ts = dtStop - dtStart;
string str = String.Format("Time elapsed for set:{0} ms", ts.TotalMilliseconds);
Logger.LogInfo(str);
_env.Dispose();
}
public void DiskSyncWriteRead()
{
var count = 5_000;
var rounds = 1;
var path = "./data/benchmark";
if (Directory.Exists(path))
{
Directory.Delete(path, true);
}
var dirS = Path.Combine(path, "Spreads");
var dirK = Path.Combine(path, "KdSoft");
Directory.CreateDirectory(dirS);
Directory.CreateDirectory(dirK);
var envS = LMDBEnvironment.Create(dirS, LMDBEnvironmentFlags.WriteMap | LMDBEnvironmentFlags.NoMetaSync,
disableAsync: true);
envS.MaxDatabases = 10;
envS.MapSize = 16 * 1024 * 1024;
envS.Open();
var dbS = envS.OpenDatabase("SimpleWrite", new DatabaseConfig(DbFlags.Create | DbFlags.IntegerKey));
for (int r = 0; r < rounds; r++)
{
using (Benchmark.Run("Spreads Write", count * 1_000_000, true))
{
for (long i = r * count; i < (r + 1) * count; i++)
{
using (var tx = envS.BeginTransaction())
{
dbS.Put(tx, i, i, TransactionPutOptions.AppendData);
tx.Commit();
}
}
}
using (Benchmark.Run("Spreads Read", count, true))
{
for (long i = r * count; i < (r + 1) * count; i++)
{
using (var tx = envS.BeginReadOnlyTransaction())
{
dbS.TryGet(tx, ref i, out long val);
if (val != i)
{
Assert.Fail();
}
}
}
}
}
dbS.Dispose();
envS.Close();
Benchmark.Dump("Writes in single OPS");
}
public LmdbEventLog(string id, string prefix, ISnapshotStore snapshotStore = null) : base(id)
{
this.id = id;
this.prefix = prefix;
this.serialization = Context.System.Serialization;
this.Settings = new LmdbEventLogSettings(Context.System.Settings.Config);
this.env = LMDBEnvironment.Create(this.Settings.RootDir);
this.db = this.env.OpenDatabase(this.Settings.DbName, new DatabaseConfig(DbFlags.Create));
this.SnapshotStore = snapshotStore ?? new LmdbSnapshotStore(this.serialization, this.env, this.db);
}
public Logic(LmdbArchiveStage <T> stage) : base(stage.Shape)
{
this.env = stage.lmdbEnvFactory();
this.inlet = stage.Inlet;
this.outlet = stage.Outlet;
this.db = env.OpenDatabase(stage.databaseName, new DatabaseConfig(DbFlags.None));
SetHandler(inlet, this);
SetHandler(outlet, this);
}
private DatabaseContext(string pathToDatabase, long mapSize, bool unsafeAsync, int hashTablesCount)
{
HashTablesCount = hashTablesCount;
// Check if current process is 64 bit one (needed for lmdb to run)
if (!Environment.Is64BitProcess)
{
throw new BadRuntimeException();
}
var lmdbOpenFlags = LMDBEnvironmentFlags.NoMemInit;
if (unsafeAsync)
{
lmdbOpenFlags = lmdbOpenFlags | LMDBEnvironmentFlags.MapAsync | LMDBEnvironmentFlags.NoLock | LMDBEnvironmentFlags.WriteMap;
}
environment = LMDBEnvironment.Create(pathToDatabase, lmdbOpenFlags, disableAsync: true);
environment.MapSize = mapSize;
environment.MaxDatabases = HashTablesCount + 5;
environment.MaxReaders = 1000;
environment.Open();
// Open all database to make sure they exists and to hold their handles
var configuration = new DatabaseConfig(DbFlags.Create | DbFlags.IntegerKey);
var tracksDatabase = environment.OpenDatabase("tracks", configuration);
var subFingerprintsDatabase = environment.OpenDatabase("subFingerprints", configuration);
var hashTables = new Database[hashTablesCount];
var hashTableConfig = new DatabaseConfig(
DbFlags.Create
| DbFlags.DuplicatesSort
| DbFlags.IntegerKey
| DbFlags.IntegerDuplicates
);
for (var i = 0; i < hashTablesCount; i++)
{
hashTables[i] = environment.OpenDatabase($"HashTable{i}", hashTableConfig);
}
databasesHolder = new DatabasesHolder(tracksDatabase, subFingerprintsDatabase, hashTables);
// Open all databases for indexes
var isrcIndex = environment.OpenDatabase("isrcIndex", new DatabaseConfig(DbFlags.Create));
var titleArtistIndex = environment.OpenDatabase("titleArtistIndex", new DatabaseConfig(DbFlags.Create | DbFlags.DuplicatesSort));
var tracksSubfingerprintsIndex = environment.OpenDatabase("tracksSubfingerprintsIndex", new DatabaseConfig(
DbFlags.Create
| DbFlags.DuplicatesSort
| DbFlags.IntegerKey
| DbFlags.IntegerDuplicates
));
indexesHolder = new IndexesHolder(isrcIndex, titleArtistIndex, tracksSubfingerprintsIndex);
}
private void SetupEnv(string path, uint mapSizeMb, LMDBEnvironmentFlags flags)
{
var env = LMDBEnvironment.Create(path, flags, disableAsync: true, disableReadTxnAutoreset: true);
env.MapSize = mapSizeMb * 1024L * 1024L;
env.MaxReaders = 1024;
env.Open();
var deadReaders = env.ReaderCheck();
if (deadReaders > 0)
{
Trace.TraceWarning($"Cleared {deadReaders} in SharedMemoryBufferPool");
}
_env = env;
}
private static LMDBEnvironment CreateEnv()
{
Logger.LogInfo("CreateEnv");
DateTime dt = DateTime.Now;
LMDBEnvironment env;
string dir = "c:\\temp\\mycache_"
+ dt.Hour.ToString() + dt.Minute.ToString() + dt.Second.ToString();
env = new LMDBEnvironment(dir);
env.MaxDatabases = 2;
env.MapSize = 10485760 * 10;
env.Open();
return(env);
}
private static void CreateAnEntry(LMDBEnvironment env)
{
Logger.LogInfo("CreateAnEntry");
var tx = env.BeginTransaction();
using (var db = tx.OpenDatabase(null,
new DatabaseConfiguration {
Flags = DatabaseOpenFlags.Create
}))
{
tx.Put(db, "hello", "world");
var result = tx.Get(db, "hello");
Logger.LogInfo("Value for key hello =>" + result);
tx.Commit();
}
}
public void CouldFindDupDSIssue()
{
Settings.UseStructLayoutSizeAsBlittableSize = true;
var env = LMDBEnvironment.Create("./Data/CouldFindDup",
LMDBEnvironmentFlags.WriteMap | LMDBEnvironmentFlags.NoSync);
env.Open();
var db = env.OpenDatabase("_streamLogs",
new DatabaseConfig(DbFlags.Create | DbFlags.IntegerDuplicates | DbFlags.DuplicatesFixed | DbFlags.IntegerKey)
{
DupSortPrefix = 64
});
// db.Truncate();
var nodupKey = 10000;
var count = 2;
for (int i = 0; i < count; i++)
{
var lr1 = new MyDupSorted()
{
Key = (ulong)i + 1
};
db.Put(nodupKey, lr1);
using (var txn = env.BeginReadOnlyTransaction())
{
var c = db.OpenReadOnlyCursor(txn);
var searchValue = new MyDupSorted()
{
Key = ulong.MaxValue
};
var found = c.TryFindDup(Lookup.LT, ref nodupKey, ref searchValue);
Assert.IsTrue(found);
c.Dispose();
}
}
db.Dispose();
env.Close();
}
/// <summary>
/// Creates a new
/// </summary>
/// <param name="directory">directory where the data and snapshots kept</param>
/// <param name="mapSize">Default is 100 MB</param>
public LmdbPersister(string directory, long mapSize = 100 * 1024 * 1024, Guid?seedId = null)
{
_directory = directory;
if (!Directory.Exists(directory))
{
Directory.CreateDirectory(directory);
}
_env = LMDBEnvironment.Create(directory, disableAsync: true);
_env.MapSize = mapSize;
_env.Open();
_snapMgr = new SnapshotManager(_directory);
_logDb = _env.OpenDatabase(Databases.Log, new DatabaseConfig(DbFlags.Create | DbFlags.DuplicatesSort)
{
DupSortPrefix = 64
});
_stateDb = _env.OpenDatabase(Databases.State, new DatabaseConfig(DbFlags.Create));
LoadState(seedId);
}
private static void EnumData(LMDBEnvironment env)
{
Logger.LogInfo("EnumData");
DateTime dtStart = DateTime.Now;
LMDBTransaction tx = env.BeginTransaction(TransactionBeginFlags.NoSync);
using (LMDBDatabase db = tx.OpenDatabase(null,
new DatabaseConfiguration {
Flags = DatabaseOpenFlags.Create
}))
{
var cur = tx.CreateCursor(db);
//cur.MoveToFirst();
int count1 = 0;
while (cur.MoveNext())
{
var fKey = Encoding.UTF8.GetString(cur.Current.Key);
var fValue = Encoding.UTF8.GetString(cur.Current.Value);
if (count1 % 10000 == 0)
{
string str3 = String.Format("count:{2} - key:{0} => value:{1}", fKey, fValue, count1);
Logger.LogInfo(str3);
}
++count1;
}
tx.Commit();
}
DateTime dtStop = DateTime.Now;
TimeSpan ts = dtStop - dtStart;
string str = String.Format("Time elapsed for enum:{0} ms", ts.TotalMilliseconds);
Logger.LogInfo(str);
}
public void CouldFindDupWideKey()
{
var path = TestUtils.GetPath();
var env = LMDBEnvironment.Create(path,
LMDBEnvironmentFlags.NoSync, disableAsync: true);
env.MapSize = 100 * 1024 * 1024;
env.Open();
var db = env.OpenDatabase("CouldFindDupWideKey",
new DatabaseConfig(DbFlags.Create | DbFlags.DuplicatesFixed)
{
DupSortPrefix = 64 * 64
});
db.Truncate();
var nodupKey = 0;
var count = 500;
{
try
{
for (int i = 1; i <= count; i++)
{
var dupValue = new DupValueWithWideKey()
{
First = (ulong)i,
Second = (ulong)i * 10 + 5,
Value = i * 10 + 5
};
db.Put(nodupKey, dupValue, TransactionPutOptions.AppendDuplicateData);
}
}
catch (Exception ex)
{
Console.WriteLine(ex);
}
}
var rounds = 1;
for (int r = 0; r < rounds; r++)
{
using (Benchmark.Run("WideKeyFind", count * 5))
{
for (int i = 1; i <= count; i++)
{
var searchValue = new DupValueWithWideKey()
{
First = 0,
Second = (ulong)i * 10, // without + 5, always smaller
Value = i * 10 + 5
};
using (var txn = env.BeginReadOnlyTransaction())
using (var c = db.OpenReadOnlyCursor(txn))
{
c.TryFindDup(Lookup.GT, ref nodupKey, ref searchValue);
if (searchValue.Value != i * 10 + 5)
{
Console.WriteLine($"2: {searchValue.Value} - {i * 10 + 5}");
Assert.Fail("Wide key doesn't work");
}
}
}
for (int i = 1; i <= count; i++)
{
var searchValue = new DupValueWithWideKey()
{
First = 0,
Second = (ulong)i * 10 + 5, // equal
Value = i * 10 + 5
};
using (var txn = env.BeginReadOnlyTransaction())
using (var c = db.OpenReadOnlyCursor(txn))
{
c.TryFindDup(Lookup.EQ, ref nodupKey, ref searchValue);
if (searchValue.Value != i * 10 + 5)
{
Console.WriteLine($"2: {searchValue.Value} - {i * 10 + 5}");
Assert.Fail("Wide key doesn't work");
}
}
searchValue = new DupValueWithWideKey()
{
First = 0,
Second = (ulong)i * 10 + 5, // equal
Value = i * 10 + 5
};
using (var txn = env.BeginReadOnlyTransaction())
using (var c = db.OpenReadOnlyCursor(txn))
{
c.TryFindDup(Lookup.GE, ref nodupKey, ref searchValue);
if (searchValue.Value != i * 10 + 5)
{
Console.WriteLine($"2: {searchValue.Value} - {i * 10 + 5}");
Assert.Fail("Wide key doesn't work");
}
}
searchValue = new DupValueWithWideKey()
{
First = 0,
Second = (ulong)i * 10 + 5, // equal
Value = i * 10 + 5
};
using (var txn = env.BeginReadOnlyTransaction())
using (var c = db.OpenReadOnlyCursor(txn))
{
if (!c.TryFindDup(Lookup.LE, ref nodupKey, ref searchValue))
{
Assert.Fail("Cannot find");
}
if (searchValue.Value != i * 10 + 5)
{
Console.WriteLine($"3: {searchValue.Second} - {i * 10 + 5}");
Assert.Fail("Wide key doesn't work");
}
}
}
for (int i = 1; i <= count; i++)
{
var searchValue = new DupValueWithWideKey()
{
First = 0,
Second = (ulong)i * 10 + 6, // +10, always larger
Value = i * 10 + 5
};
using (var txn = env.BeginReadOnlyTransaction())
using (var c = db.OpenReadOnlyCursor(txn))
{
c.TryFindDup(Lookup.LT, ref nodupKey, ref searchValue);
if (searchValue.Value != i * 10 + 5)
{
Console.WriteLine($"4: {searchValue.Value} - {i * 10 + 5}");
Assert.Fail("Wide key doesn't work");
}
}
}
}
using (Benchmark.Run("WideKeyFind SC", count * 5))
{
for (int i = 1; i <= count; i++)
{
var searchValue = new DupValueWithWideKey()
{
First = 0,
Second = (ulong)i * 10, // without + 5, always smaller
Value = i * 10 + 5
};
using (var txn = env.BeginReadOnlyTransaction())
{
db.TryFindDup(txn, Lookup.GT, ref nodupKey, ref searchValue);
if (searchValue.Value != i * 10 + 5)
{
Console.WriteLine($"2: {searchValue.Value} - {i * 10 + 5}");
Assert.Fail("Wide key doesn't work");
}
}
}
for (int i = 1; i <= count; i++)
{
var searchValue = new DupValueWithWideKey()
{
First = 0,
Second = (ulong)i * 10 + 5, // equal
Value = i * 10 + 5
};
using (var txn = env.BeginReadOnlyTransaction())
{
db.TryFindDup(txn, Lookup.EQ, ref nodupKey, ref searchValue);
if (searchValue.Value != i * 10 + 5)
{
Console.WriteLine($"2: {searchValue.Value} - {i * 10 + 5}");
Assert.Fail("Wide key doesn't work");
}
}
searchValue = new DupValueWithWideKey()
{
First = 0,
Second = (ulong)i * 10 + 5, // equal
Value = i * 10 + 5
};
using (var txn = env.BeginReadOnlyTransaction())
{
db.TryFindDup(txn, Lookup.GE, ref nodupKey, ref searchValue);
if (searchValue.Value != i * 10 + 5)
{
Console.WriteLine($"2: {searchValue.Value} - {i * 10 + 5}");
Assert.Fail("Wide key doesn't work");
}
}
searchValue = new DupValueWithWideKey()
{
First = 0,
Second = (ulong)i * 10 + 5, // equal
Value = i * 10 + 5
};
using (var txn = env.BeginReadOnlyTransaction())
{
if (!db.TryFindDup(txn, Lookup.LE, ref nodupKey, ref searchValue))
{
Assert.Fail("Cannot find");
}
if (searchValue.Value != i * 10 + 5)
{
Console.WriteLine($"3: {searchValue.Second} - {i * 10 + 5}");
Assert.Fail("Wide key doesn't work");
}
}
}
for (int i = 1; i <= count; i++)
{
var searchValue = new DupValueWithWideKey()
{
First = 0,
Second = (ulong)i * 10 + 6, // +10, always larger
Value = i * 10 + 5
};
using (var txn = env.BeginReadOnlyTransaction())
{
db.TryFindDup(txn, Lookup.LT, ref nodupKey, ref searchValue);
if (searchValue.Value != i * 10 + 5)
{
Console.WriteLine($"4: {searchValue.Value} - {i * 10 + 5}");
Assert.Fail("Wide key doesn't work");
}
}
}
}
}
Benchmark.Dump();
db.Dispose();
env.Close();
}
public void CouldFindDup()
{
var env = LMDBEnvironment.Create(TestUtils.GetPath(),
LMDBEnvironmentFlags.WriteMap | LMDBEnvironmentFlags.NoSync);
env.Open();
var db = env.OpenDatabase("CouldFindDup", new DatabaseConfig(DbFlags.Create | DbFlags.IntegerKey | DbFlags.DuplicatesFixed | DbFlags.IntegerDuplicates));
db.Truncate();
var nodupKey = 0;
var txn = env.BeginTransaction();
try
{
var midKey = 100;
using (var c = db.OpenCursor(txn))
{
Assert.IsFalse(c.TryFindDup(Lookup.LT, ref nodupKey, ref midKey));
}
using (var c = db.OpenCursor(txn))
{
Assert.IsFalse(c.TryFindDup(Lookup.LE, ref nodupKey, ref midKey));
}
using (var c = db.OpenCursor(txn))
{
Assert.IsFalse(c.TryFindDup(Lookup.EQ, ref nodupKey, ref midKey));
}
using (var c = db.OpenCursor(txn))
{
Assert.IsFalse(c.TryFindDup(Lookup.GE, ref nodupKey, ref midKey));
}
using (var c = db.OpenCursor(txn))
{
Assert.IsFalse(c.TryFindDup(Lookup.GT, ref nodupKey, ref midKey));
}
/////////////////////////////////////////////////////////////////
Assert.AreEqual(100, midKey);
db.Put(txn, nodupKey, midKey);
using (var c = db.OpenCursor(txn))
{
Assert.IsFalse(c.TryFindDup(Lookup.LT, ref nodupKey, ref midKey));
}
using (var c = db.OpenCursor(txn))
{
Assert.IsTrue(c.TryFindDup(Lookup.LE, ref nodupKey, ref midKey));
}
using (var c = db.OpenCursor(txn))
{
Assert.IsTrue(c.TryFindDup(Lookup.EQ, ref nodupKey, ref midKey));
}
using (var c = db.OpenCursor(txn))
{
Assert.IsTrue(c.TryFindDup(Lookup.GE, ref nodupKey, ref midKey));
}
using (var c = db.OpenCursor(txn))
{
Assert.IsFalse(c.TryFindDup(Lookup.GT, ref nodupKey, ref midKey));
}
/////////////////////////////////////////////////////////////////
Assert.AreEqual(100, midKey);
db.Put(txn, nodupKey, midKey + 10);
using (var c = db.OpenCursor(txn))
{
Assert.IsFalse(c.TryFindDup(Lookup.LT, ref nodupKey, ref midKey));
}
Assert.AreEqual(100, midKey);
using (var c = db.OpenCursor(txn))
{
Assert.IsTrue(c.TryFindDup(Lookup.LE, ref nodupKey, ref midKey));
}
Assert.AreEqual(100, midKey);
using (var c = db.OpenCursor(txn))
{
Assert.IsTrue(c.TryFindDup(Lookup.EQ, ref nodupKey, ref midKey));
}
Assert.AreEqual(100, midKey);
using (var c = db.OpenCursor(txn))
{
Assert.IsTrue(c.TryFindDup(Lookup.GE, ref nodupKey, ref midKey));
}
Assert.AreEqual(100, midKey);
using (var c = db.OpenCursor(txn))
{
Assert.IsTrue(c.TryFindDup(Lookup.GT, ref nodupKey, ref midKey));
}
Assert.AreEqual(110, midKey);
/////////////////////////////////////////////////////////////////
midKey = 100;
db.Put(txn, nodupKey, midKey - 10);
using (var c = db.OpenCursor(txn))
{
Assert.IsTrue(c.TryFindDup(Lookup.LT, ref nodupKey, ref midKey));
}
Assert.AreEqual(90, midKey);
midKey = 100;
using (var c = db.OpenCursor(txn))
{
Assert.IsTrue(c.TryFindDup(Lookup.LE, ref nodupKey, ref midKey));
}
Assert.AreEqual(100, midKey);
using (var c = db.OpenCursor(txn))
{
Assert.IsTrue(c.TryFindDup(Lookup.EQ, ref nodupKey, ref midKey));
}
Assert.AreEqual(100, midKey);
using (var c = db.OpenCursor(txn))
{
Assert.IsTrue(c.TryFindDup(Lookup.GE, ref nodupKey, ref midKey));
}
Assert.AreEqual(100, midKey);
using (var c = db.OpenCursor(txn))
{
Assert.IsTrue(c.TryFindDup(Lookup.GT, ref nodupKey, ref midKey));
}
Assert.AreEqual(110, midKey);
midKey = 100;
/////////////////////////////////////////////////////////////////
var bigKey = midKey + 100;
using (var c = db.OpenCursor(txn))
{
Assert.IsTrue(c.TryFindDup(Lookup.LT, ref nodupKey, ref bigKey));
Assert.AreEqual(110, bigKey);
bigKey = midKey + 100;
}
using (var c = db.OpenCursor(txn))
{
Assert.IsTrue(c.TryFindDup(Lookup.LE, ref nodupKey, ref bigKey));
Assert.AreEqual(110, bigKey);
bigKey = midKey + 100;
}
using (var c = db.OpenCursor(txn))
{
Assert.IsFalse(c.TryFindDup(Lookup.EQ, ref nodupKey, ref bigKey));
}
using (var c = db.OpenCursor(txn))
{
Assert.IsFalse(c.TryFindDup(Lookup.GE, ref nodupKey, ref bigKey));
}
using (var c = db.OpenCursor(txn))
{
Assert.IsFalse(c.TryFindDup(Lookup.GT, ref nodupKey, ref bigKey));
}
/////////////////////////////////////////////////////////////////
var smallKey = midKey - 50;
using (var c = db.OpenCursor(txn))
{
Assert.IsFalse(c.TryFindDup(Lookup.LT, ref nodupKey, ref smallKey));
}
using (var c = db.OpenCursor(txn))
{
Assert.IsFalse(c.TryFindDup(Lookup.LE, ref nodupKey, ref smallKey));
}
using (var c = db.OpenCursor(txn))
{
Assert.IsFalse(c.TryFindDup(Lookup.EQ, ref nodupKey, ref smallKey));
}
using (var c = db.OpenCursor(txn))
{
Assert.IsTrue(c.TryFindDup(Lookup.GE, ref nodupKey, ref smallKey));
Assert.AreEqual(90, smallKey);
smallKey = midKey - 50;
}
using (var c = db.OpenCursor(txn))
{
Assert.IsTrue(c.TryFindDup(Lookup.GT, ref nodupKey, ref smallKey));
Assert.AreEqual(90, smallKey);
smallKey = midKey - 50;
}
/////////////////////////////////////////////////////////////////
var midBigger = midKey + 5;
using (var c = db.OpenCursor(txn))
{
Assert.IsTrue(c.TryFindDup(Lookup.LT, ref nodupKey, ref midBigger));
Assert.AreEqual(100, midBigger);
midBigger = midKey + 5;
}
using (var c = db.OpenCursor(txn))
{
Assert.IsTrue(c.TryFindDup(Lookup.LE, ref nodupKey, ref midBigger));
Assert.AreEqual(100, midBigger);
midBigger = midKey + 5;
}
using (var c = db.OpenCursor(txn))
{
Assert.IsFalse(c.TryFindDup(Lookup.EQ, ref nodupKey, ref midBigger));
}
using (var c = db.OpenCursor(txn))
{
Assert.IsTrue(c.TryFindDup(Lookup.GE, ref nodupKey, ref midBigger));
Assert.AreEqual(110, midBigger);
midBigger = midKey + 5;
}
using (var c = db.OpenCursor(txn))
{
Assert.IsTrue(c.TryFindDup(Lookup.GT, ref nodupKey, ref midBigger));
Assert.AreEqual(110, midBigger);
midBigger = midKey + 5;
}
/////////////////////////////////////////////////////////////////
var midSmaller = midKey - 5;
using (var c = db.OpenCursor(txn))
{
Assert.IsTrue(c.TryFindDup(Lookup.LT, ref nodupKey, ref midSmaller));
Assert.AreEqual(90, midSmaller);
midSmaller = midKey - 5;
}
using (var c = db.OpenCursor(txn))
{
Assert.IsTrue(c.TryFindDup(Lookup.LE, ref nodupKey, ref midSmaller));
Assert.AreEqual(90, midSmaller);
midSmaller = midKey - 5;
}
using (var c = db.OpenCursor(txn))
{
Assert.IsFalse(c.TryFindDup(Lookup.EQ, ref nodupKey, ref midSmaller));
}
using (var c = db.OpenCursor(txn))
{
Assert.IsTrue(c.TryFindDup(Lookup.GE, ref nodupKey, ref midSmaller));
Assert.AreEqual(100, midSmaller);
midSmaller = midKey - 5;
}
using (var c = db.OpenCursor(txn))
{
Assert.IsTrue(c.TryFindDup(Lookup.GT, ref nodupKey, ref midSmaller));
Assert.AreEqual(100, midSmaller);
midSmaller = midKey - 5;
}
/////////////////////////////////////////////////////////////////
var biggerKey = midKey + 10;
using (var c = db.OpenCursor(txn))
{
Assert.IsTrue(c.TryFindDup(Lookup.LT, ref nodupKey, ref biggerKey));
Assert.AreEqual(100, biggerKey);
biggerKey = midKey + 10;
}
using (var c = db.OpenCursor(txn))
{
Assert.IsTrue(c.TryFindDup(Lookup.LE, ref nodupKey, ref biggerKey));
Assert.AreEqual(110, biggerKey);
biggerKey = midKey + 10;
}
using (var c = db.OpenCursor(txn))
{
Assert.IsTrue(c.TryFindDup(Lookup.EQ, ref nodupKey, ref biggerKey));
Assert.AreEqual(110, biggerKey);
}
using (var c = db.OpenCursor(txn))
{
Assert.IsTrue(c.TryFindDup(Lookup.GE, ref nodupKey, ref biggerKey));
Assert.AreEqual(110, biggerKey);
biggerKey = midKey + 10;
}
using (var c = db.OpenCursor(txn))
{
Assert.IsFalse(c.TryFindDup(Lookup.GT, ref nodupKey, ref biggerKey));
Assert.AreEqual(110, biggerKey);
biggerKey = midKey + 10;
}
/////////////////////////////////////////////////////////////////
var smallerKey = midKey - 10;
using (var c = db.OpenCursor(txn))
{
Assert.IsFalse(c.TryFindDup(Lookup.LT, ref nodupKey, ref smallerKey));
Assert.AreEqual(90, smallerKey);
}
using (var c = db.OpenCursor(txn))
{
Assert.IsTrue(c.TryFindDup(Lookup.LE, ref nodupKey, ref smallerKey));
Assert.AreEqual(90, smallerKey);
}
using (var c = db.OpenCursor(txn))
{
Assert.IsTrue(c.TryFindDup(Lookup.EQ, ref nodupKey, ref smallerKey));
Assert.AreEqual(90, smallerKey);
}
using (var c = db.OpenCursor(txn))
{
Assert.IsTrue(c.TryFindDup(Lookup.GE, ref nodupKey, ref smallerKey));
Assert.AreEqual(90, smallerKey);
}
using (var c = db.OpenCursor(txn))
{
Assert.IsTrue(c.TryFindDup(Lookup.GT, ref nodupKey, ref smallerKey));
Assert.AreEqual(100, smallerKey);
smallerKey = midKey - 10;
}
}
finally
{
txn.Abort();
}
db.Dispose();
env.Close();
}
public void SimpleWriteReadBenchmark()
{
#pragma warning disable 618
Settings.DoAdditionalCorrectnessChecks = false;
#pragma warning restore 618
var count = 1_00_000;
var rounds = 1;
var extraReadRounds = 10;
var path = "./data/benchmark";
if (Directory.Exists(path))
{
Directory.Delete(path, true);
}
var dirS = Path.Combine(path, "Spreads");
Directory.CreateDirectory(dirS);
var envS = LMDBEnvironment.Create(dirS, LMDBEnvironmentFlags.NoSync | LMDBEnvironmentFlags.NoLock,
disableAsync: true);
envS.MaxDatabases = 10;
envS.MapSize = 256 * 1024 * 1024;
envS.Open();
envS.TouchSpace(500);
Console.WriteLine("USED SIZE: " + envS.UsedSize);
var dbS = envS.OpenDatabase("SimpleWrite", new DatabaseConfig(DbFlags.Create | DbFlags.IntegerKey));
for (int r = 0; r < rounds; r++)
{
using (Benchmark.Run("Spreads Write (K)", count * 1000, true))
{
for (long i = r * count; i < (r + 1) * count; i++)
{
using (var tx = envS.BeginTransaction(TransactionBeginFlags.ReadWrite))
{
dbS.Put(tx, i, i, TransactionPutOptions.AppendData);
tx.Commit();
}
}
}
using (Benchmark.Run("Spreads Read", count * extraReadRounds, true))
{
for (long j = 0; j < extraReadRounds; j++)
{
for (long i = r * count; i < (r + 1) * count; i++)
{
using (var tx = envS.BeginReadOnlyTransaction())
{
dbS.TryGet(tx, ref i, out long val);
if (val != i)
{
Assert.Fail();
}
}
}
}
}
}
dbS.Dispose();
envS.Close();
Benchmark.Dump("SimpleWrite/Read 1M longs");
}
public unsafe void SimpleWriteReadBatchedBenchmark()
{
#pragma warning disable 618
Settings.DoAdditionalCorrectnessChecks = false;
#pragma warning restore 618
var count = TestUtils.GetBenchCount(TestUtils.InDocker ? 100_000 : 1_000_000, 100_000);
var rounds = 10;
var extraRounds = 10;
var path = "./data/benchmarkbatched";
if (Directory.Exists(path))
{
Directory.Delete(path, true);
}
var dirS = Path.Combine(path, "Spreads");
var dirK = Path.Combine(path, "KdSoft");
Directory.CreateDirectory(dirS);
Directory.CreateDirectory(dirK);
var envS = LMDBEnvironment.Create(dirS, LMDBEnvironmentFlags.MapAsync, disableAsync: true);
envS.MaxDatabases = 10;
envS.MapSize = 256 * 1024 * 1024;
envS.Open();
// envS.TouchSpace(100);
Console.WriteLine("USED SIZE: " + envS.UsedSize);
var dbS = envS.OpenDatabase("SimpleWrite", new DatabaseConfig(DbFlags.Create | DbFlags.IntegerKey));
for (int i = 0; i < 2; i++)
{
using (var txn = envS.BeginTransaction(TransactionBeginFlags.NoSync))
{
var key = 0L;
var keyPtr = Unsafe.AsPointer(ref key);
var key1 = new DirectBuffer(TypeHelper <int> .FixedSize, (nint)keyPtr);
var value = DirectBuffer.LengthOnly(32 * 1024 * 1024);
dbS.Put(txn, ref key1, ref value, TransactionPutOptions.ReserveSpace);
txn.Commit();
}
using (var txn = envS.BeginTransaction(TransactionBeginFlags.NoSync))
{
var key = 0L;
var keyPtr = Unsafe.AsPointer(ref key);
var key1 = new DirectBuffer(TypeHelper <int> .FixedSize, (nint)keyPtr);
dbS.Delete(txn, ref key1);
txn.Commit();
}
}
// var garbage1 = new byte[1];
for (int r = 0; r < rounds; r++)
{
using (Benchmark.Run("Spreads Write", count, false))
{
var tx = envS.BeginTransaction();
// using (tx)
{
for (long i = r * count; i < (r + 1) * count; i++)
{
dbS.Put(tx, i, i, TransactionPutOptions.AppendData);
if (i % 10000 == 0)
{
tx.Commit();
tx.Dispose();
tx = envS.BeginTransaction();
}
}
tx.Commit();
tx.Dispose();
}
}
using (Benchmark.Run("Spreads Read", count * extraRounds, false))
{
using (var tx = envS.BeginReadOnlyTransaction())
{
for (long j = 0; j < extraRounds; j++)
{
for (long i = r * count; i < (r + 1) * count; i++)
{
dbS.TryGet(tx, ref i, out long val);
if (val != i)
{
Assert.Fail();
}
}
}
}
}
}
dbS.Dispose();
envS.Close();
Benchmark.Dump("SimpleBatchedWrite/Read 10x1M longs");
// Console.WriteLine(garbage1[0]);
}
public LmdbSnapshotStore(Serialization serialization, LMDBEnvironment env, Database db)
{
this.env = env;
this.db = db;
this.snapshotSerializer = serialization.FindSerializerForType(typeof(Snapshot));
}