public void DatabaseFromCommitedTransactionShouldBeAccessable()
{
//arrange
_env.Open();
LightningDatabase db;
using (var committed = _env.BeginTransaction())
{
db = committed.OpenDatabase(null, DatabaseOpenFlags.None);
committed.Commit();
}
//act
try
{
_txn = _env.BeginTransaction();
_txn.Put(db, "key", 1);
}
finally
{
db.Close();
}
//assert
}
private void DbPut(LightningTransaction tx, LightningDatabase db, string key, DurableDataEnvelope data)
{
var byteKey = Encoding.UTF8.GetBytes(key);
var byteValue = _serializer.ToBinary(data);
tx.Put(db, byteKey, byteValue);
}
private StorageResult <TValue> StoreDocumentInternal <TKey, TValue>(ITransaction transaction, string collection, TKey key, TValue value)
{
StorageResult <TValue> result = new StorageResult <TValue>();
ValidateTransaction(transaction);
LightningTransaction lmdbTransaction = (LightningTransaction)transaction.InnerObject;
byte[] keyBytes = _environment.ConverterStore.GetToBytes <TKey>().Convert(_collectionTable[collection].Collection, key);
byte[] valueBytes = _environment.ConverterStore.GetToBytes <TValue>().Convert(_collectionTable[collection].Collection, value);
try
{
lmdbTransaction.Put(_collectionTable[collection].Collection, keyBytes, valueBytes);
long size = CaclulateSize(keyBytes, valueBytes);
((LMDBTransaction)transaction).ChangeSize(size);
_collectionTable[collection].IncrementTemporaryStats(size);
result.Document = value;
result.Status = StoreResult.Success;
return(result);
}
catch (LightningException le)
{
result.Status = HandleException(le);
return(result);
}
}
private StorageResult <TValue> UpdateDocumentInternal <TKey, TValue>(ITransaction transaction, string collection, TKey key, TValue update)
{
StorageResult <TValue> result = new StorageResult <TValue>();
ValidateTransaction(transaction);
LightningTransaction lmdbTransaction = (LightningTransaction)transaction.InnerObject;
byte[] keyBytes = _environment.ConverterStore.GetToBytes <TKey>().Convert(_collectionTable[collection].Collection, key);
byte[] oldValBytes = lmdbTransaction.Get(_collectionTable[collection].Collection, keyBytes);
long netSize = 0;
netSize = -CaclulateSize(keyBytes, oldValBytes);
byte[] valueBytes = _environment.ConverterStore.GetToBytes <TValue>().Convert(_collectionTable[collection].Collection, update);
try
{
lmdbTransaction.Put(_collectionTable[collection].Collection, keyBytes, valueBytes);
//Incase of update previous size is not being subtracted at the moment
long size = CaclulateSize(keyBytes, valueBytes);
netSize += size;
((LMDBTransaction)transaction).ChangeSize(netSize);
_collectionTable[collection].IncrementTemporaryStats(netSize);
result.Document = update;
result.Status = StoreResult.SuccessOverwrite;
return(result);
}
catch (LightningException le)
{
result.Status = HandleException(le);
return(result);
}
}
private int FailedToSend(LightningTransaction tx, OutgoingMessage message)
{
var db = OpenDatabase(tx, OutgoingQueue);
var value = tx.Get(db, message.Id.MessageIdentifier.ToByteArray());
if (value.resultCode == MDBResultCode.NotFound)
{
return(int.MaxValue);
}
var msg = value.value.CopyToNewArray().ToOutgoingMessage();
int attempts = message.SentAttempts;
if (attempts >= message.MaxAttempts)
{
RemoveMessageFromStorage(tx, OutgoingQueue, msg);
}
else if (msg.DeliverBy.HasValue)
{
var expire = msg.DeliverBy.Value;
if (expire != DateTime.MinValue && DateTime.Now >= expire)
{
RemoveMessageFromStorage(tx, OutgoingQueue, msg);
}
}
else
{
tx.Put(db, message.Id.MessageIdentifier.ToByteArray(), message.Serialize());
}
return(attempts);
}
private void SaveKV(LightningTransaction tx, byte[] key, double value)
{
var byteValue = BitConverter.GetBytes(value);
using (var db = tx.OpenDatabase("kv", new DatabaseConfiguration {
Flags = DatabaseOpenFlags.Create
}))
{
tx.Put(db, key, byteValue);
tx.Commit();
}
}
public void DatabasePutShouldNotThrowExceptions()
{
var key = "key";
var value = "value";
_txn.Put(_db, key, value);
}
public static void Index(LightningDatabase db, LightningTransaction tx, byte[] key, byte[] value)
{
// IMPORTANT:
// lmdb's DUP_SORT still imposes the MaxKeySizeBytes on the length of the key + value,
// so it's less ambiguous if we handle "multiple values for a single key" semantics ourselves
//
if (key.Length > MaxKeySizeBytes)
{
// FIXME: localize this
var message = $"Index key length is {key.Length} but the maximum key size is {MaxKeySizeBytes}";
throw new InvalidOperationException(message);
}
tx.Put(db, key, value, PutOptions.NoOverwrite);
}
public void DatabasePutShouldNotThrowExceptions()
{
var key = "key";
var value = 25;
//arrange
//act
_txn.Put(_db, key, value);
//assert
}
private void MoveToQueue(LightningTransaction tx, string queueName, Message message)
{
try
{
var idBytes = message.Id.MessageIdentifier.ToByteArray();
var original = OpenDatabase(tx, message.Queue);
var newDb = OpenDatabase(tx, queueName);
tx.Delete(original, idBytes).ThrowOnError();
tx.Put(newDb, idBytes, message.Serialize()).ThrowOnError();
}
catch (LightningException ex)
{
tx.Dispose();
if (ex.StatusCode == (int)MDBResultCode.NotFound)
{
throw new QueueDoesNotExistException("Queue doesn't exist", ex);
}
throw;
}
}
public void TruncatingTheDatabase()
{
_env.Open();
_txn = _env.BeginTransaction();
var db = _txn.OpenDatabase();
_txn.Put(db, "hello", "world");
_txn.Commit();
_txn.Dispose();
_txn = _env.BeginTransaction();
db = _txn.OpenDatabase();
db.Truncate(_txn);
_txn.Commit();
_txn.Dispose();
_txn = _env.BeginTransaction();
db = _txn.OpenDatabase();
var result = _txn.Get(db, UTF8.GetBytes("hello"));
Assert.Null(result);
}
public void TruncatingTheDatabase()
{
_env.Open();
_txn = _env.BeginTransaction();
var db = _txn.OpenDatabase();
_txn.Put(db, "hello", "world");
_txn.Commit();
_txn.Dispose();
_txn = _env.BeginTransaction();
db = _txn.OpenDatabase();
db.Truncate(_txn);
_txn.Commit();
_txn.Dispose();
_txn = _env.BeginTransaction();
db = _txn.OpenDatabase();
var result = _txn.Get(db, UTF8.GetBytes("hello"));
Assert.Equal(MDBResultCode.NotFound, result.resultCode);
}
private void StoreIncomingMessages(LightningTransaction tx, params Message[] messages)
{
try
{
foreach (var messagesByQueue in messages.GroupBy(x => x.Queue))
{
var db = OpenDatabase(tx, messagesByQueue.Key);
foreach (var message in messagesByQueue)
{
tx.Put(db, message.Id.MessageIdentifier.ToByteArray(), message.Serialize()).ThrowOnError();
}
}
}
catch (LightningException ex)
{
if (ex.StatusCode == (int)MDBResultCode.NotFound)
{
throw new QueueDoesNotExistException("Queue doesn't exist", ex);
}
throw;
}
}
public static bool PutVulnerabilities(List <OSSIndexVulnerability> vulnerabilities)
{
if (!Environment.IsOpened)
{
Environment.MapSize = 2L * 1024L * 1024L * 1024L;
Environment.Open(EnvironmentOpenFlags.None);
}
using (LightningTransaction tx = Environment.BeginTransaction())
using (LightningDatabase db = tx.OpenDatabase(null, new DatabaseConfiguration {
Flags = DatabaseOpenFlags.Create
}))
{
try
{
foreach (OSSIndexVulnerability v in vulnerabilities)
{
/*
* using (MemoryStream ms = new MemoryStream())
* using (BsonWriter writer = new BsonWriter(ms))
* {
* JsonSerializer serializer = new JsonSerializer();
* serializer.Serialize(writer, v);
* }
*/
tx.Put(db, Encoding.UTF8.GetBytes(string.IsNullOrEmpty(v.Vid) ? v.Url : v.Vid + "#" + v.Group), Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(v)));
}
tx.Commit();
logger.Debug("Committed {0} vulnerabilities to database.", vulnerabilities.Count);
return(true);
}
catch (LightningException e)
{
logger.Error("Exception thrown attempting to write vulnerabilities to database.");
logger.Error(e);
return(false);
}
}
}
public static MDBResultCode Put(this LightningTransaction tx, LightningDatabase db, string key, string value)
{
var enc = System.Text.Encoding.UTF8;
return(tx.Put(db, enc.GetBytes(key), enc.GetBytes(value)));
}
public void TruncatingTheDatabase()
{
_env.Open();
_txn = _env.BeginTransaction();
var db = _txn.OpenDatabase();
_txn.Put(db, "hello", "world");
_txn.Commit();
_txn.Dispose();
_txn = _env.BeginTransaction();
db = _txn.OpenDatabase();
db.Truncate(_txn);
_txn.Commit();
_txn.Dispose();
_txn = _env.BeginTransaction();
db = _txn.OpenDatabase();
var result = _txn.Get(db, UTF8.GetBytes("hello"));
Assert.Null(result);
}
public void DatabaseFromCommitedTransactionShouldBeAccessable()
{
//arrange
_env.Open();
LightningDatabase db;
using (var committed = _env.BeginTransaction())
{
db = committed.OpenDatabase(null, DatabaseOpenFlags.None);
committed.Commit();
}
//act
try
{
_txn = _env.BeginTransaction();
_txn.Put(db, "key", 1);
}
finally
{
db.Close();
}
//assert
}
public void Put(TKey key, T data) => _tx.Put(_db, _settings.SerializeKey(key), _settings.Serialize(data), PutOptions.NoDuplicateData);
private void StoreOutgoing(LightningTransaction tx, OutgoingMessage message)
{
var db = OpenDatabase(tx, OutgoingQueue);
tx.Put(db, message.Id.MessageIdentifier.ToByteArray(), message.Serialize());
}
public void CanCountTransactionEntries()
{
//arrange
_txn = _env.BeginTransaction();
var db = _txn.OpenDatabase(null, new DatabaseOptions { Flags = DatabaseOpenFlags.None });
const int entriesCount = 10;
for (var i = 0; i < entriesCount; i++)
_txn.Put(db, i.ToString(), i.ToString());
//act
var count = _txn.GetEntriesCount(db);
//assert;
Assert.AreEqual(entriesCount, count);
}
public void TransactionShouldSupportCustomComparer()
{
//arrange
Func<int, int, int> comparison = (l, r) => -Math.Sign(l - r);
_txn = _env.BeginTransaction();
var db = _txn.OpenDatabase(
options: new DatabaseOptions { Compare = b => b.FromFunc(comparison) });
var keysUnsorted = new int[] { 2, 10, 5 };
var keysSorted = keysUnsorted.ToArray();
Array.Sort(keysSorted, new Comparison<int>(comparison));
//act
for (var i = 0; i < keysUnsorted.Length; i++)
_txn.Put(keysUnsorted[i], i);
//assert
using (var c = _txn.CreateCursor(db))
{
int order = 0;
KeyValuePair<int, int> pair;
while (c.MoveNext(out pair))
Assert.AreEqual(keysSorted[order++], pair.Key);
}
}
public void Put(byte[] key, byte[] value)
{
_tx.Put(_db, key, value);
}
public static void Put(this LightningTransaction tx, LightningDatabase db, string key, string value)
{
var enc = System.Text.Encoding.UTF8;
tx.Put(db, enc.GetBytes(key), enc.GetBytes(value));
}