public void TestIteratorWithSnapshot()
{
using (var db = new DB(new Options { CreateIfMissing = true }, "test2.db"))
{
using (var batch = new WriteBatch())
{
for (int i = 0; i < 100; i++)
{
batch.Put(Slice.FromString($"key::{i,20:D20}"), Slice.FromString($"{i,32}"));
}
db.Write(new WriteOptions { Sync = true }, batch);
}
using (var snapshot = db.GetSnapshot())
using (var batch = new WriteBatch())
using (var itr = db.NewIterator(new ReadOptions { Snapshot = snapshot }))
{
itr.Seek(Slice.FromString("key::"));
Assert.IsTrue(itr.Valid());
int entriesDeleted = 0;
while (itr.Valid())
{
batch.Delete(itr.Key());
itr.Next();
entriesDeleted++;
}
db.Write(new WriteOptions(), batch);
Assert.AreEqual(100, entriesDeleted);
}
}
}
public async Task SnapshotTest()
{
using (var storage = await NewStorageAsync())
{
var str1 = "test1";
var str2 = "test2";
var s1 = new MemoryStream(Encoding.UTF8.GetBytes(str1));
var s2 = new MemoryStream(Encoding.UTF8.GetBytes(str2));
var writeBatch = new WriteBatch();
writeBatch.Put("key1", s1);
await storage.Writer.WriteAsync(writeBatch);
var snapshot = storage.Commands.CreateSnapshot();
writeBatch = new WriteBatch();
writeBatch.Put("key1", s2);
await storage.Writer.WriteAsync(writeBatch);
AssertEqual(str2, storage.Reader.Read("key1"));
AssertEqual(str1, storage.Reader.Read("key1", new ReadOptions
{
Snapshot = snapshot
}));
storage.Commands.ReleaseSnapshot(snapshot);
}
}
public async Task ShouldRecoverDataFromLogFile()
{
var storage = await NewStorageAsync();
var name = storage.Name;
var writeBatch = new WriteBatch();
writeBatch.Put("A", new MemoryStream(Encoding.UTF8.GetBytes("123")));
writeBatch.Put("B", new MemoryStream(Encoding.UTF8.GetBytes("123")));
writeBatch.Put("D", new MemoryStream(Encoding.UTF8.GetBytes("123")));
storage.Writer.WriteAsync(writeBatch).Wait();
var fileSystem = storage.StorageState.FileSystem;
storage.Dispose();
using (var newStorage = new Storage(new StorageState(name, new StorageOptions())
{
FileSystem = fileSystem
}))
{
await newStorage.InitAsync();
using(var it = newStorage.Reader.NewIterator(new ReadOptions()))
{
it.Seek("C");
Assert.True(it.IsValid);
it.Prev();
Assert.True(it.IsValid);
Assert.Equal("B", it.Key);
}
}
}
public void ReadVersion_Items_From_Both_WriteBatch_And_Snapshot_WithoutVersionNumber()
{
using (var tx = Env.NewTransaction(TransactionFlags.ReadWrite))
{
Env.CreateTree(tx, "tree");
tx.Environment.CreateTree(tx,"tree").Add("foo1", StreamFor("foo1"));
tx.Commit();
}
using (var writeBatch = new WriteBatch())
using (var snapshot = Env.CreateSnapshot())
{
writeBatch.Add("foo2", StreamFor("foo2"), "tree");
var foor1Version = snapshot.ReadVersion("tree", "foo1", writeBatch);
var foo2Version = snapshot.ReadVersion("tree", "foo2", writeBatch);
var foo2VersionThatShouldBe0 = snapshot.ReadVersion("tree", "foo2");
Assert.Equal(1, foor1Version);
Assert.Equal(0, foo2Version); //added to write batch without version number, so 0 is version number that is fetched
Assert.Equal(0, foo2VersionThatShouldBe0);
}
}
public void ReadVersion_Items_From_Both_WriteBatch_And_Snapshot()
{
using (var tx = Env.NewTransaction(TransactionFlags.ReadWrite))
{
Env.CreateTree(tx, "tree");
tx.Environment.CreateTree(tx,"tree").Add("foo1", StreamFor("foo1"));
tx.Commit();
}
using (var writeBatch = new WriteBatch())
using (var snapshot = Env.CreateSnapshot())
{
writeBatch.Add("foo2", StreamFor("foo2"), "tree", 1);
var foor1Version = snapshot.ReadVersion("tree", "foo1", writeBatch);
var foo2Version = snapshot.ReadVersion("tree", "foo2", writeBatch);
var foo2VersionThatShouldBe0 = snapshot.ReadVersion("tree", "foo2");
Assert.Equal(1, foor1Version);
Assert.Equal(2, foo2Version); //is not committed yet
Assert.Equal(0, foo2VersionThatShouldBe0);
}
}
public async Task CanOpenAndCloseWithUpdate()
{
var storage = await NewStorageAsync();
var name = storage.Name;
var str1 = "test1";
var str2 = "test2";
var s1 = new MemoryStream(Encoding.UTF8.GetBytes(str1));
var s2 = new MemoryStream(Encoding.UTF8.GetBytes(str2));
var writeBatch = new WriteBatch();
writeBatch.Put("A", s1);
await storage.Writer.WriteAsync(writeBatch);
writeBatch = new WriteBatch();
writeBatch.Put("A", s2);
await storage.Writer.WriteAsync(writeBatch);
var fileSystem = storage.StorageState.FileSystem;
storage.Dispose();
using (var newStorage = new Storage(new StorageState(name, new StorageOptions())
{
FileSystem = fileSystem
}))
{
await newStorage.InitAsync();
AssertEqual(str2, newStorage.Reader.Read("A"));
}
}
public void Read_Items_From_Both_WriteBatch_And_Snapshot()
{
using (var tx = Env.NewTransaction(TransactionFlags.ReadWrite))
{
Env.CreateTree(tx, "tree");
tx.Environment.CreateTree(tx,"tree").Add("foo1", StreamFor("foo1"));
tx.Commit();
}
using (var writeBatch = new WriteBatch())
using (var snapshot = Env.CreateSnapshot())
{
writeBatch.Add("foo2", StreamFor("foo2"), "tree");
var foo1ReadResult = snapshot.Read("tree", "foo1", writeBatch);
var foo2ReadResult = snapshot.Read("tree", "foo2", writeBatch);
var foo2ReadResultThatShouldBeNull = snapshot.Read("tree", "foo2");
Assert.NotNull(foo1ReadResult);
Assert.NotNull(foo2ReadResult);
Assert.Null(foo2ReadResultThatShouldBeNull);
Assert.Equal(foo1ReadResult.Reader.ToStringValue(), "foo1");
Assert.Equal(foo2ReadResult.Reader.ToStringValue(), "foo2");
}
}
public void CanAddMultiValuesUnderTheSameKeyToBatch()
{
using (var env = new StorageEnvironment(StorageEnvironmentOptions.CreateMemoryOnly()))
{
var rand = new Random();
var testBuffer = new byte[168];
rand.NextBytes(testBuffer);
CreateTrees(env, 1, "multitree");
var batch = new WriteBatch();
batch.MultiAdd("key", "value1", "multitree0");
batch.MultiAdd("key", "value2", "multitree0");
env.Writer.Write(batch);
using (var tx = env.NewTransaction(TransactionFlags.Read))
{
var tree = tx.Environment.CreateTree(tx,"multitree0");
using (var it = tree.MultiRead("key"))
{
Assert.True(it.Seek(Slice.BeforeAllKeys));
Assert.Equal("value1", it.CurrentKey.ToString());
Assert.True(it.MoveNext());
Assert.Equal("value2", it.CurrentKey.ToString());
}
}
}
}
public async Task CanGetLatestVersion()
{
var storage = await NewStorageAsync();
var name = storage.Name;
for (int j = 0; j < 3; j++)
{
for (int i = 0; i < 65; i++)
{
for (int k = 0; k < 4; k++)
{
var writeBatch = new WriteBatch();
writeBatch.Put("A" + k, new MemoryStream(new[] { (byte)i, (byte)k }));
await storage.Writer.WriteAsync(writeBatch);
}
}
}
var fileSystem = storage.StorageState.FileSystem;
storage.Dispose();
using (var newStorage = new Storage(new StorageState(name, new StorageOptions())
{
FileSystem = fileSystem
}))
{
await newStorage.InitAsync();
var stream = newStorage.Reader.Read("A2");
Assert.Equal(64, stream.ReadByte());
}
}
public void Should_be_able_to_read_and_write_lots_of_data()
{
CreatTestSchema();
var writeBatch = new WriteBatch();
var testData = GenerateTestData().ToList();
foreach (var dataPair in testData)
writeBatch.Add(dataPair.Key, StreamFor(dataPair.Value), TestTreeName);
Env.Writer.Write(writeBatch);
using (var snapshot = Env.CreateSnapshot())
{
using (var iterator = snapshot.Iterate(TestTreeName))
{
Assert.True(iterator.Seek(Slice.BeforeAllKeys));
do
{
var value = iterator.CreateReaderForCurrent().ToStringValue();
var extractedDataPair = new KeyValuePair<string, string>(iterator.CurrentKey.ToString(), value);
Assert.Contains(extractedDataPair,testData);
} while (iterator.MoveNext());
}
}
}
public ReadResult Read(string treeName, Slice key, WriteBatch writeBatch = null)
{
Tree tree = null;
if (writeBatch != null)
{
WriteBatch.BatchOperationType operationType;
Stream stream;
ushort? version;
if (writeBatch.TryGetValue(treeName, key, out stream, out version, out operationType))
{
if (!version.HasValue)
tree = GetTree(treeName);
switch (operationType)
{
case WriteBatch.BatchOperationType.Add:
{
var reader = new ValueReader(stream);
return new ReadResult(reader, version.HasValue ? (ushort)(version.Value + 1) : tree.ReadVersion(key));
}
case WriteBatch.BatchOperationType.Delete:
return null;
}
}
}
if (tree == null)
tree = GetTree(treeName);
return tree.Read(key);
}
public bool Contains(string treeName, Slice key, out ushort? version, WriteBatch writeBatch = null)
{
if (writeBatch != null)
{
WriteBatch.BatchOperationType operationType;
Stream stream;
if (writeBatch.TryGetValue(treeName, key, out stream, out version, out operationType))
{
switch (operationType)
{
case WriteBatch.BatchOperationType.Add:
return true;
case WriteBatch.BatchOperationType.Delete:
return false;
default:
throw new ArgumentOutOfRangeException(operationType.ToString());
}
}
}
var tree = GetTree(treeName);
var readVersion = tree.ReadVersion(key);
var exists = readVersion > 0;
version = exists ? (ushort?)readVersion : null;
return exists;
}
public void SimpleIncrementShouldWorkUsingWriteBatch()
{
CreateTrees(Env, 1, "tree");
var writeBatch = new WriteBatch();
writeBatch.Increment("key/1", 10, "tree0");
Env.Writer.Write(writeBatch);
writeBatch = new WriteBatch();
writeBatch.Increment("key/1", 5, "tree0");
Env.Writer.Write(writeBatch);
writeBatch = new WriteBatch();
writeBatch.Increment("key/1", -3, "tree0");
Env.Writer.Write(writeBatch);
using (var tx = Env.NewTransaction(TransactionFlags.Read))
{
var read = tx.ReadTree("tree0").Read("key/1");
Assert.NotNull(read);
Assert.Equal(3, read.Version);
Assert.Equal(12, read.Reader.ReadLittleEndianInt64());
}
}
public override void Accept(Disk d)
{
var ms = new MemoryStream();
_serializer.Serialize(new JsonTextWriter(new StreamWriter(ms)), d);
ms.Position = 0;
var key = new Slice(EndianBitConverter.Big.GetBytes(counter++));
_currentBatch.Add(key, ms, "albums");
foreach (var diskId in d.DiskIds)
{
_currentBatch.MultiAdd(diskId, key, "ix_diskids");
}
if(d.Artist != null)
_currentBatch.MultiAdd(d.Artist.ToLower(), key, "ix_artists");
if (d.Title != null)
_currentBatch.MultiAdd(d.Title.ToLower(), key, "ix_titles");
if (counter%1000 == 0)
{
_storageEnvironment.Writer.Write(_currentBatch);
_currentBatch = new WriteBatch();
}
}
public virtual void Add(WriteBatch writeBatch, Slice key, RavenJToken value, ushort? expectedVersion = null)
{
var stream = new BufferPoolMemoryStream(BufferPool);
value.WriteTo(stream);
stream.Position = 0;
writeBatch.Add(key, stream, TableName, expectedVersion);
}
public virtual void Add(WriteBatch writeBatch, string key, byte[] value, ushort? expectedVersion = null)
{
var stream = new BufferPoolMemoryStream(BufferPool);
stream.Write(value, 0, value.Length);
stream.Position = 0;
writeBatch.Add(key, stream, TableName, expectedVersion);
}
public async Task WillNotFindMissingValue()
{
using (var storage = await NewStorageAsync(new StorageOptions()))
{
var writeBatch = new WriteBatch();
writeBatch.Put("system/config", new MemoryStream());
await storage.Writer.WriteAsync(writeBatch);
Assert.Null(storage.Reader.Read("system/Test"));
}
}
public async Task ReadFromMemTable()
{
using (var storage = await NewStorageAsync())
{
var batch = new WriteBatch();
batch.Put("test1", new MemoryStream(Encoding.UTF8.GetBytes("test")));
await storage.Writer.WriteAsync(batch);
Assert.NotNull(storage.Reader.Read("test1"));
}
}
public void Write(WriteBatch writeBatch)
{
try
{
env.Writer.Write(writeBatch);
}
catch (AggregateException ae)
{
if (ae.InnerException is OperationCanceledException == false) // this can happen during storage disposal
throw;
}
}
public void SingleItemBatchTest()
{
var batch = new WriteBatch();
batch.Add("key/1", new MemoryStream(Encoding.UTF8.GetBytes("123")), Constants.RootTreeName);
Env.Writer.Write(batch);
using (var snapshot = Env.CreateSnapshot())
{
var reader = snapshot.Read(null, "key/1").Reader;
Assert.Equal("123", reader.ToStringValue());
}
}
public VoronDisksDestination()
{
_storageEnvironment = new StorageEnvironment(StorageEnvironmentOptions.ForPath("FreeDB"));
using (var tx = _storageEnvironment.NewTransaction(TransactionFlags.ReadWrite))
{
_storageEnvironment.CreateTree(tx, "albums");
_storageEnvironment.CreateTree(tx, "ix_diskids");
_storageEnvironment.CreateTree(tx, "ix_artists");
_storageEnvironment.CreateTree(tx, "ix_titles");
tx.Commit();
}
_currentBatch = new WriteBatch();
}
public void Should_be_able_to_read_and_write_small_values()
{
CreatTestSchema();
var writeBatch = new WriteBatch();
writeBatch.Add("key",StreamFor("value"),TestTreeName);
Env.Writer.Write(writeBatch);
using (var snapshot = Env.CreateSnapshot())
{
var storedValue = Encoding.UTF8.GetString(snapshot.Read(TestTreeName, "key").Reader.AsStream().ReadData());
Assert.Equal("value",storedValue);
}
}
protected RavenJObject LoadJson(Table table, Slice key, WriteBatch writeBatch, out ushort version)
{
var read = table.Read(Snapshot, key, writeBatch);
if (read == null)
{
version = table.ReadVersion(Snapshot, key, writeBatch) ?? 0;
return null;
}
using (var stream = read.Reader.AsStream())
{
version = read.Version;
return stream.ToJObject();
}
}
private static Task BatchWriteRangeInTask(string dbName, int offset, int limit)
{
return Task.Run(() =>
{
using (var db = new DB(new Options { CreateIfMissing = true }, dbName))
using (var batch = new WriteBatch())
{
for (int i = offset; i < limit; i++)
{
batch.Put(Slice.FromString($"key::{i,20:D20}"), Slice.FromString($"{i,32}"));
}
db.Write(new WriteOptions { Sync = true }, batch);
}
});
}
public void Write(WriteBatch batch)
{
if (batch.IsEmpty)
return;
EnsureValidBackgroundTaskState();
using (var mine = new OutstandingWrite(batch, this))
{
_pendingWrites.Enqueue(mine);
_hasWrites.Set();
mine.Wait();
}
}
public async Task ReadFromImmutableMemTable()
{
using (var storage = await NewStorageAsync(new StorageOptions
{
WriteBatchSize = 1
}))
{
var batch1 = new WriteBatch();
batch1.Put("test1", new MemoryStream(Encoding.UTF8.GetBytes("test")));
await storage.Writer.WriteAsync(batch1);
var batch2 = new WriteBatch();
batch2.Put("test2", new MemoryStream(Encoding.UTF8.GetBytes("test")));
await storage.Writer.WriteAsync(batch2);
Assert.NotNull(storage.Reader.Read("test1"));
}
}
public void SetupDatabaseIdAndSchemaVersion()
{
using (var snapshot = storage.CreateSnapshot())
{
Guid id;
string schemaVersion;
Slice idKey = new Slice ("id");
Slice schemaVersionKey = new Slice("schema_version");
var read = storage.Details.Read(snapshot, idKey, null);
if (read == null) // new db
{
id = Guid.NewGuid();
schemaVersion = SchemaVersion;
using (var writeIdBatch = new WriteBatch())
{
storage.Details.Add(writeIdBatch, idKey, id.ToByteArray());
storage.Details.Add(writeIdBatch, schemaVersionKey, schemaVersion);
storage.Write(writeIdBatch);
}
}
else
{
if (read.Reader.Length != 16) //precaution - might prevent NRE in edge cases
throw new InvalidDataException("Failed to initialize Voron transactional storage. Possible data corruption. (no db id)");
using (var stream = read.Reader.AsStream())
using (var reader = new BinaryReader(stream))
{
id = new Guid(reader.ReadBytes((int)stream.Length));
}
var schemaRead = storage.Details.Read(snapshot, schemaVersionKey, null);
if (schemaRead == null)
throw new InvalidDataException("Failed to initialize Voron transactional storage. Possible data corruption. (no schema version)");
schemaVersion = schemaRead.Reader.ToStringValue();
}
storage.SetDatabaseIdAndSchemaVersion(id, schemaVersion);
}
}
public void BatchConcurrencyExceptionShouldBeThrownWhenVersionMismatch()
{
var batch1 = new WriteBatch();
batch1.Add("key/1", StreamFor("123"), Constants.RootTreeName, 0);
Env.Writer.Write(batch1);
var batch2 = new WriteBatch();
batch2.Add("key/1", StreamFor("123"), Constants.RootTreeName, 2);
var e = Assert.Throws<AggregateException>(() => Env.Writer.Write(batch2)).InnerException;
Assert.Equal("Cannot add 'key/1' to 'Root' tree. Version mismatch. Expected: 2. Actual: 1.", e.Message);
var batch3 = new WriteBatch();
batch3.Delete("key/1", Constants.RootTreeName, 2);
e = Assert.Throws<AggregateException>(() => Env.Writer.Write(batch3)).InnerException;
Assert.Equal("Cannot delete 'key/1' to 'Root' tree. Version mismatch. Expected: 2. Actual: 1.", e.Message);
}
public void BatchConcurrencyExceptionShouldNotBeThrown2()
{
var batch1 = new WriteBatch();
batch1.Add("key/1", StreamFor("123"), Constants.RootTreeName, 0);
Env.Writer.Write(batch1);
using (var snapshot = Env.CreateSnapshot())
{
var version = snapshot.ReadVersion(Constants.RootTreeName, "key/1", batch1);
Assert.Equal(1, version);
batch1 = new WriteBatch();
batch1.Delete("key/1", Constants.RootTreeName);
version = snapshot.ReadVersion(Constants.RootTreeName, "key/1", batch1);
batch1.Add("key/1", StreamFor("123"), Constants.RootTreeName, version);
Env.Writer.Write(batch1);
}
}
public void WriteSomethingToVoron()
{
var serializer = new JsonSerializer();
using (var storage = new StorageEnvironment(StorageEnvironmentOptions.GetInMemory()))
{
using (var tx = storage.NewTransaction(TransactionFlags.ReadWrite))
{
storage.CreateTree(tx, "foos");
tx.Commit();
}
{
var ms = new MemoryStream();
var batch = new WriteBatch();
var foo = new Foo { Id = "hello", Value = 99 };
using (var writer = new StreamWriter(ms))
{
serializer.Serialize(new JsonTextWriter(writer), foo);
writer.Flush();
ms.Position = 0;
//var key = new Slice(EndianBitConverter.Big.GetBytes(counter++));
batch.Add(foo.Id, ms, "foos");
storage.Writer.Write(batch);
}
}
using (var tx = storage.NewTransaction(TransactionFlags.Read))
{
var foos = tx.GetTree("foos");
var readResult = foos.Read(tx, "hello");
using (var stream = readResult.Reader.AsStream())
{
var foo = serializer.Deserialize<Foo>(new JsonTextReader(new StreamReader(stream)));
Assert.Equal(99, foo.Value);
}
}
}
}
public virtual ushort?ReadVersion(SnapshotReader snapshot, Slice key, WriteBatch writeBatch)
{
return(snapshot.ReadVersion(TableName, key, writeBatch));
}
public virtual void Add(WriteBatch writeBatch, string key, byte[] value, ushort?expectedVersion = null)
{
Add(writeBatch, (Slice)key, value, expectedVersion);
}
public virtual void MultiAdd(WriteBatch writeBatch, Slice key, Slice value, ushort?expectedVersion = null)
{
AssertKey(key);
writeBatch.MultiAdd(key, value, TableName, expectedVersion);
}
public RocksDbAtomicWrite(IRocksDbContext context)
{
_context = context;
_writeBatch = new WriteBatch();
}
public virtual void Delete(WriteBatch writeBatch, Slice key, ushort?expectedVersion = null, bool shouldIgnoreConcurrencyExceptions = false)
{
AssertKey(key);
writeBatch.Delete(key, TableName, expectedVersion, shouldIgnoreConcurrencyExceptions);
}
private void Persist(Block block)
{
WriteBatch batch = new WriteBatch();
DbCache <UInt160, AccountState> accounts = new DbCache <UInt160, AccountState>(db, DataEntryPrefix.ST_Account, batch);
DbCache <UInt256, UnspentCoinState> unspentcoins = new DbCache <UInt256, UnspentCoinState>(db, DataEntryPrefix.ST_Coin, batch);
DbCache <UInt256, SpentCoinState> spentcoins = new DbCache <UInt256, SpentCoinState>(db, DataEntryPrefix.ST_SpentCoin, batch);
DbCache <ECPoint, ValidatorState> validators = new DbCache <ECPoint, ValidatorState>(db, DataEntryPrefix.ST_Validator, batch);
DbCache <UInt256, AssetState> assets = new DbCache <UInt256, AssetState>(db, DataEntryPrefix.ST_Asset, batch);
DbCache <UInt160, ContractState> contracts = new DbCache <UInt160, ContractState>(db, DataEntryPrefix.ST_Contract, batch);
DbCache <StorageKey, StorageItem> storages = new DbCache <StorageKey, StorageItem>(db, DataEntryPrefix.ST_Storage, batch);
DbMetaDataCache <ValidatorsCountState> validators_count = new DbMetaDataCache <ValidatorsCountState>(db, DataEntryPrefix.IX_ValidatorsCount);
long amount_sysfee = GetSysFeeAmount(block.PrevHash) + (long)block.Transactions.Sum(p => p.SystemFee);
batch.Put(SliceBuilder.Begin(DataEntryPrefix.DATA_Block).Add(block.Hash), SliceBuilder.Begin().Add(amount_sysfee).Add(block.Trim()));
foreach (Transaction tx in block.Transactions)
{
batch.Put(SliceBuilder.Begin(DataEntryPrefix.DATA_Transaction).Add(tx.Hash), SliceBuilder.Begin().Add(block.Index).Add(tx.ToArray()));
unspentcoins.Add(tx.Hash, new UnspentCoinState
{
Items = Enumerable.Repeat(CoinState.Confirmed, tx.Outputs.Length).ToArray()
});
foreach (TransactionOutput output in tx.Outputs)
{
AccountState account = accounts.GetAndChange(output.ScriptHash, () => new AccountState(output.ScriptHash));
if (account.Balances.ContainsKey(output.AssetId))
{
account.Balances[output.AssetId] += output.Value;
}
else
{
account.Balances[output.AssetId] = output.Value;
}
if (output.AssetId.Equals(GoverningToken.Hash) && account.Votes.Length > 0)
{
foreach (ECPoint pubkey in account.Votes)
{
validators.GetAndChange(pubkey, () => new ValidatorState(pubkey)).Votes += output.Value;
}
validators_count.GetAndChange().Votes[account.Votes.Length - 1] += output.Value;
}
}
foreach (var group in tx.Inputs.GroupBy(p => p.PrevHash))
{
Transaction tx_prev = GetTransaction(ReadOptions.Default, group.Key, out int height);
foreach (CoinReference input in group)
{
unspentcoins.GetAndChange(input.PrevHash).Items[input.PrevIndex] |= CoinState.Spent;
TransactionOutput out_prev = tx_prev.Outputs[input.PrevIndex];
AccountState account = accounts.GetAndChange(out_prev.ScriptHash);
if (out_prev.AssetId.Equals(GoverningToken.Hash))
{
spentcoins.GetAndChange(input.PrevHash, () => new SpentCoinState
{
TransactionHash = input.PrevHash,
TransactionHeight = (uint)height,
Items = new Dictionary <ushort, uint>()
}).Items.Add(input.PrevIndex, block.Index);
if (account.Votes.Length > 0)
{
foreach (ECPoint pubkey in account.Votes)
{
ValidatorState validator = validators.GetAndChange(pubkey);
validator.Votes -= out_prev.Value;
if (!validator.Registered && validator.Votes.Equals(Fixed8.Zero))
{
validators.Delete(pubkey);
}
}
validators_count.GetAndChange().Votes[account.Votes.Length - 1] -= out_prev.Value;
}
}
account.Balances[out_prev.AssetId] -= out_prev.Value;
}
}
switch (tx)
{
#pragma warning disable CS0612
case RegisterTransaction tx_register:
assets.Add(tx.Hash, new AssetState
{
AssetId = tx_register.Hash,
AssetType = tx_register.AssetType,
Name = tx_register.Name,
Amount = tx_register.Amount,
Available = Fixed8.Zero,
Precision = tx_register.Precision,
Fee = Fixed8.Zero,
FeeAddress = new UInt160(),
Owner = tx_register.Owner,
Admin = tx_register.Admin,
Issuer = tx_register.Admin,
Expiration = block.Index + 2 * 2000000,
IsFrozen = false
});
break;
#pragma warning restore CS0612
case IssueTransaction _:
foreach (TransactionResult result in tx.GetTransactionResults().Where(p => p.Amount < Fixed8.Zero))
{
assets.GetAndChange(result.AssetId).Available -= result.Amount;
}
break;
case ClaimTransaction _:
foreach (CoinReference input in ((ClaimTransaction)tx).Claims)
{
if (spentcoins.TryGet(input.PrevHash)?.Items.Remove(input.PrevIndex) == true)
{
spentcoins.GetAndChange(input.PrevHash);
}
}
break;
#pragma warning disable CS0612
case EnrollmentTransaction tx_enrollment:
validators.GetAndChange(tx_enrollment.PublicKey, () => new ValidatorState(tx_enrollment.PublicKey)).Registered = true;
break;
#pragma warning restore CS0612
case StateTransaction tx_state:
foreach (StateDescriptor descriptor in tx_state.Descriptors)
{
switch (descriptor.Type)
{
case StateType.Account:
ProcessAccountStateDescriptor(descriptor, accounts, validators, validators_count);
break;
case StateType.Validator:
ProcessValidatorStateDescriptor(descriptor, validators);
break;
}
}
break;
#pragma warning disable CS0612
case PublishTransaction tx_publish:
contracts.GetOrAdd(tx_publish.ScriptHash, () => new ContractState
{
Script = tx_publish.Script,
ParameterList = tx_publish.ParameterList,
ReturnType = tx_publish.ReturnType,
ContractProperties = (ContractPropertyState)Convert.ToByte(tx_publish.NeedStorage),
Name = tx_publish.Name,
CodeVersion = tx_publish.CodeVersion,
Author = tx_publish.Author,
Email = tx_publish.Email,
Description = tx_publish.Description
});
break;
#pragma warning restore CS0612
case InvocationTransaction tx_invocation:
CachedScriptTable script_table = new CachedScriptTable(contracts);
using (StateMachine service = new StateMachine(block, accounts, assets, contracts, storages))
{
ApplicationEngine engine = new ApplicationEngine(TriggerType.Application, tx_invocation, script_table, service, tx_invocation.Gas);
engine.LoadScript(tx_invocation.Script, false);
if (engine.Execute())
{
service.Commit();
}
ApplicationExecuted?.Invoke(this, new ApplicationExecutedEventArgs(tx_invocation, service.Notifications.ToArray(), engine));
}
break;
}
}
accounts.DeleteWhere((k, v) => !v.IsFrozen && v.Votes.Length == 0 && v.Balances.All(p => p.Value <= Fixed8.Zero));
accounts.Commit();
unspentcoins.DeleteWhere((k, v) => v.Items.All(p => p.HasFlag(CoinState.Spent)));
unspentcoins.Commit();
spentcoins.DeleteWhere((k, v) => v.Items.Count == 0);
spentcoins.Commit();
validators.Commit();
assets.Commit();
contracts.Commit();
storages.Commit();
validators_count.Commit(batch);
batch.Put(SliceBuilder.Begin(DataEntryPrefix.SYS_CurrentBlock), SliceBuilder.Begin().Add(block.Hash).Add(block.Index));
db.Write(WriteOptions.Default, batch);
current_block_height = block.Index;
}
public virtual void MultiDelete(WriteBatch writeBatch, Slice key, Slice value, ushort?expectedVersion = null)
{
writeBatch.MultiDelete(key, value, TableName, expectedVersion);
}
public TreeIterator Iterate(WriteBatch writeBatch = null)
{
return(new TreeIterator(this, _tx));
}
public void Record_debug_journal_and_replay_it()
{
using (var env = new StorageEnvironment(StorageEnvironmentOptions.CreateMemoryOnly()))
{
env.DebugJournal = new DebugJournal("debug_journal_test", env, true);
using (var tx = env.NewTransaction(TransactionFlags.ReadWrite))
{
env.CreateTree(tx, "test-tree");
tx.Commit();
}
using (var writeBatch = new WriteBatch())
{
var valueBuffer = new MemoryStream(Encoding.UTF8.GetBytes("testing testing 1!"));
writeBatch.Add("foo", valueBuffer, "test-tree");
valueBuffer = new MemoryStream(Encoding.UTF8.GetBytes("testing testing 1 2!"));
writeBatch.Add("bar", valueBuffer, "test-tree");
valueBuffer = new MemoryStream(Encoding.UTF8.GetBytes("testing testing 1 2 3!"));
writeBatch.Add("foo-bar", valueBuffer, "test-tree");
writeBatch.MultiAdd("multi-foo", "AA", "test-tree");
env.Writer.Write(writeBatch);
}
using (var writeBatch = new WriteBatch())
{
writeBatch.MultiAdd("multi-foo", "BB", "test-tree");
writeBatch.MultiAdd("multi-foo", "CC", "test-tree");
writeBatch.Delete("foo-bar", "test-tree");
env.Writer.Write(writeBatch);
}
using (var tx = env.NewTransaction(TransactionFlags.ReadWrite))
{
env.CreateTree(tx, "test-tree2");
tx.Commit();
}
using (var writeBatch = new WriteBatch())
{
var valueBuffer = new MemoryStream(Encoding.UTF8.GetBytes("testing testing 1!"));
writeBatch.Add("foo", valueBuffer, "test-tree2");
valueBuffer = new MemoryStream(Encoding.UTF8.GetBytes("testing testing 1 2!"));
writeBatch.Add("bar", valueBuffer, "test-tree2");
valueBuffer = new MemoryStream(Encoding.UTF8.GetBytes("testing testing 1 2 3!"));
writeBatch.Add("foo-bar", valueBuffer, "test-tree2");
env.Writer.Write(writeBatch);
}
}
using (var env = new StorageEnvironment(StorageEnvironmentOptions.CreateMemoryOnly()))
{
env.DebugJournal = DebugJournal.FromFile("debug_journal_test", env);
env.DebugJournal.Replay();
using (var snapshot = env.CreateSnapshot())
{
Assert.Equal("testing testing 1!", snapshot.Read("test-tree", "foo").Reader.ToStringValue());
Assert.Equal("testing testing 1 2!", snapshot.Read("test-tree", "bar").Reader.ToStringValue());
Assert.Equal("testing testing 1!", snapshot.Read("test-tree2", "foo").Reader.ToStringValue());
Assert.Equal("testing testing 1 2!", snapshot.Read("test-tree2", "bar").Reader.ToStringValue());
Assert.Equal("testing testing 1 2 3!", snapshot.Read("test-tree2", "foo-bar").Reader.ToStringValue());
Assert.Equal(0, snapshot.ReadVersion("test-tree", "foo-bar"));
using (var iter = snapshot.MultiRead("test-tree", "multi-foo"))
{
iter.Seek(Slice.BeforeAllKeys);
Assert.Equal("AA", iter.CurrentKey.ToString());
Assert.DoesNotThrow(() => iter.MoveNext());
Assert.Equal("BB", iter.CurrentKey.ToString());
Assert.DoesNotThrow(() => iter.MoveNext());
Assert.Equal("CC", iter.CurrentKey.ToString());
}
}
}
}
public virtual void Add(WriteBatch writeBatch, Slice key, string value, ushort?expectedVersion = null)
{
Add(writeBatch, key, Encoding.UTF8.GetBytes(value), expectedVersion);
}
protected StructureReader <T> LoadStruct <T>(TableOfStructures <T> table, Slice key, WriteBatch writeBatch, out ushort version)
{
var read = table.ReadStruct(Snapshot, key, writeBatch);
if (read == null)
{
version = 0;
return(null);
}
version = read.Version;
return(read.Reader);
}
public bool Contains(SnapshotReader snapshot, Slice key, WriteBatch writeBatch)
{
ushort?version;
return(Contains(snapshot, key, writeBatch, out version));
}
public virtual IIterator Iterate(SnapshotReader snapshot, WriteBatch writeBatch)
{
return(snapshot.Iterate(TableName));
}
public void CommitBatch()
{
_db.Write(_currentBatch);
_currentBatch.Dispose();
_currentBatch = null;
}
public virtual void Delete(WriteBatch writeBatch, Slice key, ushort?expectedVersion = null)
{
writeBatch.Delete(key, TableName, expectedVersion);
}
public virtual void Add(WriteBatch writeBatch, Slice key, Stream value, ushort?expectedVersion = null, bool shouldIgnoreConcurrencyExceptions = false)
{
AssertKey(key);
writeBatch.Add(key, value, TableName, expectedVersion, shouldIgnoreConcurrencyExceptions);
}
public virtual ReadResult Read(SnapshotReader snapshot, Slice key, WriteBatch writeBatch)
{
return(snapshot.Read(TableName, key, writeBatch));
}
public virtual void Delete(WriteBatch writeBatch, string key, ushort?expectedVersion = null)
{
Delete(writeBatch, (Slice)key, expectedVersion);
}
public LevelDBBlockchain(string path)
{
header_index.Add(GenesisBlock.Hash);
Version version;
Slice value;
db = DB.Open(path, new Options {
CreateIfMissing = true
});
if (db.TryGet(ReadOptions.Default, SliceBuilder.Begin(DataEntryPrefix.SYS_Version), out value) && Version.TryParse(value.ToString(), out version) && version >= Version.Parse("2.7.4"))
{
ReadOptions options = new ReadOptions {
FillCache = false
};
value = db.Get(options, SliceBuilder.Begin(DataEntryPrefix.SYS_CurrentBlock));
UInt256 current_header_hash = new UInt256(value.ToArray().Take(32).ToArray());
this.current_block_height = value.ToArray().ToUInt32(32);
uint current_header_height = current_block_height;
if (db.TryGet(options, SliceBuilder.Begin(DataEntryPrefix.SYS_CurrentHeader), out value))
{
current_header_hash = new UInt256(value.ToArray().Take(32).ToArray());
current_header_height = value.ToArray().ToUInt32(32);
}
foreach (UInt256 hash in db.Find(options, SliceBuilder.Begin(DataEntryPrefix.IX_HeaderHashList), (k, v) =>
{
using (MemoryStream ms = new MemoryStream(v.ToArray(), false))
using (BinaryReader r = new BinaryReader(ms))
{
return(new
{
Index = k.ToArray().ToUInt32(1),
Hashes = r.ReadSerializableArray <UInt256>()
});
}
}).OrderBy(p => p.Index).SelectMany(p => p.Hashes).ToArray())
{
if (!hash.Equals(GenesisBlock.Hash))
{
header_index.Add(hash);
}
stored_header_count++;
}
if (stored_header_count == 0)
{
Header[] headers = db.Find(options, SliceBuilder.Begin(DataEntryPrefix.DATA_Block), (k, v) => Header.FromTrimmedData(v.ToArray(), sizeof(long))).OrderBy(p => p.Index).ToArray();
for (int i = 1; i < headers.Length; i++)
{
header_index.Add(headers[i].Hash);
}
}
else if (current_header_height >= stored_header_count)
{
for (UInt256 hash = current_header_hash; hash != header_index[(int)stored_header_count - 1];)
{
Header header = Header.FromTrimmedData(db.Get(options, SliceBuilder.Begin(DataEntryPrefix.DATA_Block).Add(hash)).ToArray(), sizeof(long));
header_index.Insert((int)stored_header_count, hash);
hash = header.PrevHash;
}
}
}
else
{
WriteBatch batch = new WriteBatch();
ReadOptions options = new ReadOptions {
FillCache = false
};
using (Iterator it = db.NewIterator(options))
{
for (it.SeekToFirst(); it.Valid(); it.Next())
{
batch.Delete(it.Key());
}
}
db.Write(WriteOptions.Default, batch);
Persist(GenesisBlock);
db.Put(WriteOptions.Default, SliceBuilder.Begin(DataEntryPrefix.SYS_Version), GetType().GetTypeInfo().Assembly.GetName().Version.ToString());
}
thread_persistence = new Thread(PersistBlocks);
thread_persistence.Name = "LevelDBBlockchain.PersistBlocks";
thread_persistence.Start();
}
private void ProcessBlock(Block block, HashSet <UInt160> accounts, WriteBatch batch)
{
foreach (Transaction tx in block.Transactions)
{
HashSet <UInt160> accounts_changed = new HashSet <UInt160>();
for (ushort index = 0; index < tx.Outputs.Length; index++)
{
TransactionOutput output = tx.Outputs[index];
if (accounts_tracked.ContainsKey(output.ScriptHash))
{
CoinReference reference = new CoinReference
{
PrevHash = tx.Hash,
PrevIndex = index
};
if (coins_tracked.TryGetValue(reference, out Coin coin))
{
coin.State |= CoinState.Confirmed;
}
else
{
accounts_tracked[output.ScriptHash].Add(reference);
coins_tracked.Add(reference, coin = new Coin
{
Reference = reference,
Output = output,
State = CoinState.Confirmed
});
}
batch.Put(SliceBuilder.Begin(DataEntryPrefix.ST_Coin).Add(reference), SliceBuilder.Begin().Add(output).Add((byte)coin.State));
accounts_changed.Add(output.ScriptHash);
}
}
foreach (CoinReference input in tx.Inputs)
{
if (coins_tracked.TryGetValue(input, out Coin coin))
{
if (coin.Output.AssetId.Equals(Blockchain.GoverningToken.Hash))
{
coin.State |= CoinState.Spent | CoinState.Confirmed;
batch.Put(SliceBuilder.Begin(DataEntryPrefix.ST_Coin).Add(input), SliceBuilder.Begin().Add(coin.Output).Add((byte)coin.State));
}
else
{
accounts_tracked[coin.Output.ScriptHash].Remove(input);
coins_tracked.Remove(input);
batch.Delete(DataEntryPrefix.ST_Coin, input);
}
accounts_changed.Add(coin.Output.ScriptHash);
}
}
switch (tx)
{
case MinerTransaction _:
case ContractTransaction _:
#pragma warning disable CS0612
case PublishTransaction _:
#pragma warning restore CS0612
break;
case ClaimTransaction tx_claim:
foreach (CoinReference claim in tx_claim.Claims)
{
if (coins_tracked.TryGetValue(claim, out Coin coin))
{
accounts_tracked[coin.Output.ScriptHash].Remove(claim);
coins_tracked.Remove(claim);
batch.Delete(DataEntryPrefix.ST_Coin, claim);
accounts_changed.Add(coin.Output.ScriptHash);
}
}
break;
#pragma warning disable CS0612
case EnrollmentTransaction tx_enrollment:
if (accounts_tracked.ContainsKey(tx_enrollment.ScriptHash))
{
accounts_changed.Add(tx_enrollment.ScriptHash);
}
break;
case RegisterTransaction tx_register:
if (accounts_tracked.ContainsKey(tx_register.OwnerScriptHash))
{
accounts_changed.Add(tx_register.OwnerScriptHash);
}
break;
#pragma warning restore CS0612
default:
foreach (UInt160 hash in tx.Witnesses.Select(p => p.ScriptHash))
{
if (accounts_tracked.ContainsKey(hash))
{
accounts_changed.Add(hash);
}
}
break;
}
if (accounts_changed.Count > 0)
{
foreach (UInt160 account in accounts_changed)
{
batch.Put(SliceBuilder.Begin(DataEntryPrefix.ST_Transaction).Add(account).Add(tx.Hash), false);
}
WalletTransaction?.Invoke(null, new WalletTransactionEventArgs
{
Transaction = tx,
RelatedAccounts = accounts_changed.ToArray(),
Height = block.Index,
Time = block.Timestamp
});
}
}
}
public void ParallelWritesInBatchesAndReadsByUsingTreeIterator()
{
const int numberOfWriteThreads = 10;
const int numberOfReadThreads = 10;
const int numberOfTrees = 2;
var trees = CreateTrees(Env, numberOfTrees, "tree");
Task readParallelTask = null;
var taskWorkTime = TimeSpan.FromSeconds(60);
var writeTime = Stopwatch.StartNew();
var writeParallelTask = Task.Factory.StartNew(
() =>
{
Parallel.For(
0,
numberOfWriteThreads,
i =>
{
var random = new Random(i ^ 1337);
var dataSize = random.Next(100, 100);
var buffer = new byte[dataSize];
random.NextBytes(buffer);
while (writeTime.Elapsed < taskWorkTime && (readParallelTask == null || readParallelTask.Exception == null))
{
var tIndex = random.Next(0, numberOfTrees - 1);
var treeName = trees[tIndex];
var batch = new WriteBatch();
batch.Add("testdocuments/" + random.Next(0, 100000), new MemoryStream(buffer), treeName);
Env.Writer.Write(batch);
}
});
},
TaskCreationOptions.LongRunning);
var readTime = Stopwatch.StartNew();
readParallelTask = Task.Factory.StartNew(
() =>
{
Parallel.For(
0,
numberOfReadThreads,
i =>
{
var random = new Random(i);
while (readTime.Elapsed < taskWorkTime)
{
var tIndex = random.Next(0, numberOfTrees - 1);
var treeName = trees[tIndex];
using (var snapshot = Env.CreateSnapshot())
using (var iterator = snapshot.Iterate(treeName))
{
if (!iterator.Seek(Slice.BeforeAllKeys))
{
continue;
}
do
{
Assert.Contains("testdocuments/", iterator.CurrentKey.ToString());
} while (iterator.MoveNext());
}
}
});
},
TaskCreationOptions.LongRunning);
try
{
Task.WaitAll(new[] { writeParallelTask, readParallelTask });
}
catch (Exception ex)
{
var aggregate = ex as AggregateException;
if (aggregate != null)
{
foreach (var innerEx in aggregate.InnerExceptions)
{
Console.WriteLine(innerEx);
}
}
throw ex;
}
}
public virtual void Increment(WriteBatch writeBatch, Slice key, long delta, ushort?expectedVersion = null)
{
AssertKey(key);
writeBatch.Increment(key, delta, TableName, expectedVersion);
}
public void StartBatch()
{
_currentBatch = new WriteBatch();
}
private void Persist(Block block, WriteBatch writeBatch, bool addIntoCache = false)
{
if (writeBatch == null)
{
throw new ArgumentNullException(nameof(writeBatch));
}
if (block == null)
{
throw new ArgumentNullException(nameof(block));
}
var defaultCallValue = new DataWord(new byte[] { 0x00 });
foreach (var transaction in block.Transactions)
{
var smartContractTransaction = transaction as SmartContractTransaction;
if (smartContractTransaction == null)
{
continue;
}
if (smartContractTransaction.To == null || !smartContractTransaction.To.Any()) // Create the contract.
{
var solidityVm = new SolidityVm();
var program = new SolidityProgram(smartContractTransaction.Data.ToList(), new SolidityProgramInvoke(new byte[0], new DataWord(smartContractTransaction.From.ToArray()), defaultCallValue, this));
while (!program.IsStopped())
{
solidityVm.Step(program);
}
var contractCode = program.GetResult().GetHReturn();
var txId = transaction.GetTxId();
var smartContractAdr = smartContractTransaction.GetSmartContractAddress();
var hex = contractCode.ToHexString();
writeBatch.Put(string.Format(SMART_CONTRACT_ELT, smartContractAdr.ToHexString()), hex);
writeBatch.Put(string.Format(SMART_CONTRACT_TX_ELT, txId.ToHexString()), smartContractAdr.ToHexString());
if (addIntoCache)
{
_cacheSmartContracts.Add(smartContractAdr.ToHexString(), new SmartContract
{
Address = smartContractAdr,
Code = contractCode
});
_cacheTxSmartContracts.Add(txId.ToHexString(), new SmartContract
{
Address = smartContractAdr,
Code = contractCode
});
}
}
else if (smartContractTransaction.To != null)
{
var sc = GetSmartContract(smartContractTransaction.To);
if (sc == null)
{
return;
}
var from = smartContractTransaction.From;
if (from == null)
{
from = new byte[0];
}
var solidityVm = new SolidityVm();
var program = new SolidityProgram(sc.Code.ToList(), new SolidityProgramInvoke(smartContractTransaction.Data, smartContractTransaction.To, new DataWord(from.ToArray()), defaultCallValue, this));
while (!program.IsStopped())
{
solidityVm.Step(program);
}
var logs = program.GetResult().GetLogs();
if (logs != null && logs.Any())
{
AddLogs(smartContractTransaction.To, block.GetHashHeader(), logs, addIntoCache);
}
}
}
}
public WriteBatchWrapper(WriteBatch writeBatch)
{
_writeBatch = writeBatch ?? throw new ArgumentNullException(nameof(writeBatch));
}
public bool Contains(SnapshotReader snapshot, Slice key, WriteBatch writeBatch, out ushort?version)
{
return(snapshot.Contains(TableName, key, out version, writeBatch));
}
public void Rollback()
{
_writeBatch = new WriteBatch();
ClearCache();
}
/// <summary>
/// This should be used when writing to the RocksDbStore to ensure that the
/// default write option settings are used whenever writes occur.
/// </summary>
/// <param name="writeBatch"></param>
private void WriteInternal(WriteBatch writeBatch)
{
m_store.Write(writeBatch, m_defaults.WriteOptions);
}
public WriteBatchWrapper(WriteBatch writeBatch)
{
_wrapped = writeBatch;
}
public void FunctionalTest()
{
string temp = Path.GetTempPath();
var testdir = Path.Combine(temp, "functional_test");
var testdb = Path.Combine(testdir, "main");
var testcp = Path.Combine(testdir, "cp");
var path = Environment.ExpandEnvironmentVariables(testdb);
var cppath = Environment.ExpandEnvironmentVariables(testcp);
if (Directory.Exists(testdir))
{
Directory.Delete(testdir, true);
}
Directory.CreateDirectory(testdir);
var options = new DbOptions()
.SetCreateIfMissing(true)
.EnableStatistics();
// Using standard open
using (var db = RocksDb.Open(options, path))
{
// With strings
string value = db.Get("key");
db.Put("key", "value");
Assert.Equal("value", db.Get("key"));
Assert.Null(db.Get("non-existent-key"));
db.Remove("key");
Assert.Null(db.Get("value"));
// With bytes
db.Put(Encoding.UTF8.GetBytes("key"), Encoding.UTF8.GetBytes("value"));
Assert.True(BinaryComparer.Default.Equals(Encoding.UTF8.GetBytes("value"), db.Get(Encoding.UTF8.GetBytes("key"))));
// non-existent kiey
Assert.Null(db.Get(new byte[] { 0, 1, 2 }));
db.Remove(Encoding.UTF8.GetBytes("key"));
Assert.Null(db.Get(Encoding.UTF8.GetBytes("key")));
db.Put(Encoding.UTF8.GetBytes("key"), new byte[] { 0, 1, 2, 3, 4, 5, 6, 7 });
// With buffers
var buffer = new byte[100];
long length = db.Get(Encoding.UTF8.GetBytes("key"), buffer, 0, buffer.Length);
Assert.Equal(8, length);
Assert.Equal(new byte[] { 0, 1, 2, 3, 4, 5, 6, 7 }, buffer.Take((int)length).ToList());
buffer = new byte[5];
length = db.Get(Encoding.UTF8.GetBytes("key"), buffer, 0, buffer.Length);
Assert.Equal(8, length);
Assert.Equal(new byte[] { 0, 1, 2, 3, 4 }, buffer.Take((int)Math.Min(buffer.Length, length)));
length = db.Get(Encoding.UTF8.GetBytes("bogus"), buffer, 0, buffer.Length);
Assert.Equal(-1, length);
// Write batches
// With strings
using (WriteBatch batch = new WriteBatch()
.Put("one", "uno")
.Put("two", "deuce")
.Put("two", "dos")
.Put("three", "tres"))
{
db.Write(batch);
}
Assert.Equal("uno", db.Get("one"));
// With save point
using (WriteBatch batch = new WriteBatch())
{
batch
.Put("hearts", "red")
.Put("diamonds", "red");
batch.SetSavePoint();
batch
.Put("clubs", "black");
batch.SetSavePoint();
batch
.Put("spades", "black");
batch.RollbackToSavePoint();
db.Write(batch);
}
Assert.Equal("red", db.Get("diamonds"));
Assert.Equal("black", db.Get("clubs"));
Assert.Null(db.Get("spades"));
// Save a checkpoint
using (var cp = db.Checkpoint())
{
cp.Save(cppath);
}
// With bytes
var utf8 = Encoding.UTF8;
using (WriteBatch batch = new WriteBatch()
.Put(utf8.GetBytes("four"), new byte[] { 4, 4, 4 })
.Put(utf8.GetBytes("five"), new byte[] { 5, 5, 5 }))
{
db.Write(batch);
}
Assert.True(BinaryComparer.Default.Equals(new byte[] { 4, 4, 4 }, db.Get(utf8.GetBytes("four"))));
// Snapshots
using (var snapshot = db.CreateSnapshot())
{
var before = db.Get("one");
db.Put("one", "1");
var useSnapshot = new ReadOptions()
.SetSnapshot(snapshot);
// the database value was written
Assert.Equal("1", db.Get("one"));
// but the snapshot still sees the old version
var after = db.Get("one", readOptions: useSnapshot);
Assert.Equal(before, after);
}
var two = db.Get("two");
Assert.Equal("dos", two);
// Iterators
using (var iterator = db.NewIterator(
readOptions: new ReadOptions()
.SetIterateUpperBound("t")
))
{
iterator.Seek("k");
Assert.True(iterator.Valid());
Assert.Equal("key", iterator.StringKey());
iterator.Next();
Assert.True(iterator.Valid());
Assert.Equal("one", iterator.StringKey());
Assert.Equal("1", iterator.StringValue());
iterator.Next();
Assert.False(iterator.Valid());
}
// MultiGet
var multiGetResult = db.MultiGet(new[] { "two", "three", "nine" });
Assert.Equal(
expected: new[]
{
new KeyValuePair <string, string>("two", "dos"),
new KeyValuePair <string, string>("three", "tres"),
new KeyValuePair <string, string>("nine", null)
},
actual: multiGetResult
);
}
// Test reading checkpointed db
using (var cpdb = RocksDb.Open(options, cppath))
{
Assert.Equal("red", cpdb.Get("diamonds"));
Assert.Equal("black", cpdb.Get("clubs"));
Assert.Null(cpdb.Get("spades"));
// Checkpoint occurred before these changes:
Assert.Null(cpdb.Get("four"));
}
// Test various operations
using (var db = RocksDb.Open(options, path))
{
// Nulls should be allowed here
db.CompactRange((byte[])null, (byte[])null);
db.CompactRange((string)null, (string)null);
}
// Test with column families
var optionsCf = new DbOptions()
.SetCreateIfMissing(true)
.SetCreateMissingColumnFamilies(true);
var columnFamilies = new ColumnFamilies
{
{ "reverse", new ColumnFamilyOptions() },
};
using (var db = RocksDb.Open(optionsCf, path, columnFamilies))
{
var reverse = db.GetColumnFamily("reverse");
db.Put("one", "uno");
db.Put("two", "dos");
db.Put("three", "tres");
db.Put("uno", "one", cf: reverse);
db.Put("dos", "two", cf: reverse);
db.Put("tres", "three", cf: reverse);
}
// Test Cf Delete
using (var db = RocksDb.Open(optionsCf, path, columnFamilies))
{
var reverse = db.GetColumnFamily("reverse");
db.Put("cuatro", "four", cf: reverse);
db.Put("cinco", "five", cf: reverse);
Assert.Equal("four", db.Get("cuatro", cf: reverse));
Assert.Equal("five", db.Get("cinco", cf: reverse));
byte[] keyBytes = Encoding.UTF8.GetBytes("cuatro");
db.Remove(keyBytes, reverse);
db.Remove("cinco", reverse);
Assert.Null(db.Get("cuatro", cf: reverse));
Assert.Null(db.Get("cinco", cf: reverse));
}
// Test list
{
var list = RocksDb.ListColumnFamilies(optionsCf, path);
Assert.Equal(new[] { "default", "reverse" }, list.ToArray());
}
// Test reopen with column families
using (var db = RocksDb.Open(optionsCf, path, columnFamilies))
{
var reverse = db.GetColumnFamily("reverse");
Assert.Equal("uno", db.Get("one"));
Assert.Equal("one", db.Get("uno", cf: reverse));
Assert.Null(db.Get("uno"));
Assert.Null(db.Get("one", cf: reverse));
}
// Test dropping and creating column family
using (var db = RocksDb.Open(options, path, columnFamilies))
{
db.DropColumnFamily("reverse");
var reverse = db.CreateColumnFamily(new ColumnFamilyOptions(), "reverse");
Assert.Null(db.Get("uno", cf: reverse));
db.Put("red", "rouge", cf: reverse);
Assert.Equal("rouge", db.Get("red", cf: reverse));
}
// Test reopen after drop and create
using (var db = RocksDb.Open(options, path, columnFamilies))
{
var reverse = db.GetColumnFamily("reverse");
Assert.Null(db.Get("uno", cf: reverse));
Assert.Equal("rouge", db.Get("red", cf: reverse));
}
// Test read only
using (var db = RocksDb.OpenReadOnly(options, path, columnFamilies, false))
{
Assert.Equal("uno", db.Get("one"));
}
// Test SstFileWriter
{
using (var writer = new SstFileWriter())
{
}
var envOpts = new EnvOptions();
var ioOpts = new ColumnFamilyOptions();
using (var sst = new SstFileWriter(envOpts, ioOpts))
{
var filename = Path.Combine(temp, "test.sst");
if (File.Exists(filename))
{
File.Delete(filename);
}
sst.Open(filename);
sst.Add("four", "quatro");
sst.Add("one", "uno");
sst.Add("two", "dos");
sst.Finish();
using (var db = RocksDb.Open(options, path, columnFamilies))
{
Assert.NotEqual("four", db.Get("four"));
var ingestOptions = new IngestExternalFileOptions()
.SetMoveFiles(true);
db.IngestExternalFiles(new string[] { filename }, ingestOptions);
Assert.Equal("quatro", db.Get("four"));
}
}
}
// test comparator
unsafe
{
var opts = new ColumnFamilyOptions()
.SetComparator(new IntegerStringComparator());
var filename = Path.Combine(temp, "test.sst");
if (File.Exists(filename))
{
File.Delete(filename);
}
using (var sst = new SstFileWriter(ioOptions: opts))
{
sst.Open(filename);
sst.Add("111", "111");
sst.Add("1001", "1001"); // this order is only allowed using an integer comparator
sst.Finish();
}
}
// test write batch with index
{
var wbwi = new WriteBatchWithIndex(reservedBytes: 1024);
wbwi.Put("one", "un");
wbwi.Put("two", "deux");
var oneValueIn = Encoding.UTF8.GetBytes("one");
var oneValueOut = wbwi.Get("one");
Assert.Equal("un", oneValueOut);
using (var db = RocksDb.Open(options, path, columnFamilies))
{
var oneCombinedOut = wbwi.Get(db, "one");
var threeCombinedOut = wbwi.Get(db, "three");
Assert.Equal("un", oneCombinedOut);
Assert.Equal("tres", threeCombinedOut);
using (var wbIterator = wbwi.NewIterator(db.NewIterator()))
{
wbIterator.Seek("o");
Assert.True(wbIterator.Valid());
var itkey = wbIterator.StringKey();
Assert.Equal("one", itkey);
var itval = wbIterator.StringValue();
Assert.Equal("un", itval);
wbIterator.Next();
Assert.True(wbIterator.Valid());
itkey = wbIterator.StringKey();
Assert.Equal("three", itkey);
itval = wbIterator.StringValue();
Assert.Equal("tres", itval);
wbIterator.Next();
Assert.True(wbIterator.Valid());
itkey = wbIterator.StringKey();
Assert.Equal("two", itkey);
itval = wbIterator.StringValue();
Assert.Equal("deux", itval);
wbIterator.Next();
Assert.False(wbIterator.Valid());
}
db.Write(wbwi);
var oneDbOut = wbwi.Get("one");
Assert.Equal("un", oneDbOut);
}
}
// compact range
{
using (var db = RocksDb.Open(options, path, columnFamilies))
{
db.CompactRange("o", "tw");
}
}
// Test that GC does not cause access violation on Comparers
{
if (Directory.Exists("test-av-error"))
{
Directory.Delete("test-av-error", true);
}
options = new RocksDbSharp.DbOptions()
.SetCreateIfMissing(true)
.SetCreateMissingColumnFamilies(true);
var sc = new RocksDbSharp.StringComparator(StringComparer.InvariantCultureIgnoreCase);
columnFamilies = new RocksDbSharp.ColumnFamilies
{
{ "cf1", new RocksDbSharp.ColumnFamilyOptions()
.SetComparator(sc) },
};
GC.Collect();
using (var db = RocksDbSharp.RocksDb.Open(options, "test-av-error", columnFamilies))
{
}
if (Directory.Exists("test-av-error"))
{
Directory.Delete("test-av-error", true);
}
}
// Smoke test various options
{
var dbname = "test-options";
if (Directory.Exists(dbname))
{
Directory.Delete(dbname, true);
}
var optsTest = (DbOptions) new RocksDbSharp.DbOptions()
.SetCreateIfMissing(true)
.SetCreateMissingColumnFamilies(true)
.SetBlockBasedTableFactory(new BlockBasedTableOptions().SetBlockCache(Cache.CreateLru(1024 * 1024)));
GC.Collect();
using (var db = RocksDbSharp.RocksDb.Open(optsTest, dbname))
{
}
if (Directory.Exists(dbname))
{
Directory.Delete(dbname, true);
}
}
// Smoke test OpenWithTtl
{
var dbname = "test-with-ttl";
if (Directory.Exists(dbname))
{
Directory.Delete(dbname, true);
}
var optsTest = (DbOptions) new RocksDbSharp.DbOptions()
.SetCreateIfMissing(true)
.SetCreateMissingColumnFamilies(true);
using (var db = RocksDbSharp.RocksDb.OpenWithTtl(optsTest, dbname, 1))
{
}
if (Directory.Exists(dbname))
{
Directory.Delete(dbname, true);
}
}
// Smoke test MergeOperator
{
var dbname = "test-merge-operator";
if (Directory.Exists(dbname))
{
Directory.Delete(dbname, true);
}
var optsTest = (DbOptions) new RocksDbSharp.DbOptions()
.SetCreateIfMissing(true)
.SetMergeOperator(MergeOperators.Create(
name: "test-merge-operator",
partialMerge: (key, keyLength, operandsList, operandsListLength, numOperands, success, newValueLength) => IntPtr.Zero,
fullMerge: (key, keyLength, existingValue, existingValueLength, operandsList, operandsListLength, numOperands, success, newValueLength) => IntPtr.Zero,
deleteValue: (value, valueLength) => { }
));
GC.Collect();
using (var db = RocksDbSharp.RocksDb.Open(optsTest, dbname))
{
}
if (Directory.Exists(dbname))
{
Directory.Delete(dbname, true);
}
}
// Test that GC does not cause access violation on Comparers
{
var dbname = "test-av-error";
if (Directory.Exists(dbname))
{
Directory.Delete(dbname, true);
}
options = new RocksDbSharp.DbOptions()
.SetCreateIfMissing(true)
.SetCreateMissingColumnFamilies(true);
var sc = new RocksDbSharp.StringComparator(StringComparer.InvariantCultureIgnoreCase);
columnFamilies = new RocksDbSharp.ColumnFamilies
{
{ "cf1", new RocksDbSharp.ColumnFamilyOptions()
.SetComparator(sc) },
};
GC.Collect();
using (var db = RocksDbSharp.RocksDb.Open(options, dbname, columnFamilies))
{
}
if (Directory.Exists(dbname))
{
Directory.Delete(dbname, true);
}
}
}
