public DBPartition(Logger logger, string fileName)
{
this.partitionName = Path.GetFileName(fileName);
this.path = Path.GetDirectoryName(fileName);
this.logger = logger;
if (string.IsNullOrEmpty(path))
{
throw new Exception("Rocksdb storage path was not configured properly");
}
if (!path.EndsWith("/"))
{
path += '/';
}
//logger.Message($"RocksDB partition path: {fileName}");
this._db = RocksDbStore.Instance(logger, path);
// Create partition if it doesn't exist already
try
{
logger.Message("Getting partition: " + this.partitionName);
this.partition = this._db.GetColumnFamily(partitionName);
}
catch
{
logger.Message("Partition not found, create it now: " + this.partitionName);
var cf = new ColumnFamilyOptions();
// TODO different partitions might need different options...
this.partition = this._db.CreateColumnFamily(cf, partitionName);
}
}
public RocksDbColumnFamiliesCommandTest()
{
_temporaryDirectory = Path.Combine(
Path.GetTempPath(),
Guid.NewGuid().ToString());
_columnFamilies = Enumerable.Range(0, 10)
.Select(_ => Guid.NewGuid())
.ToList();
using (var db = OpenRocksDb(_temporaryDirectory))
{
var columnFamilyOptions = new ColumnFamilyOptions();
foreach (var columnFamily in _columnFamilies)
{
db.CreateColumnFamily(
columnFamilyOptions,
columnFamily.ToString());
}
}
_rocksDbService = new RocksDbService();
_stringInputOutputErrorContainer = new StringInputOutputErrorContainer(
new StringReader(string.Empty),
new StringWriter(),
new StringWriter());
_command = new RocksDbColumnFamiliesCommand(
_stringInputOutputErrorContainer,
_rocksDbService);
_stringInputOutputErrorContainer.SetNewLines();
}
public void RocksDbOptionsProviderCreateCfOptions()
{
var env32 = new Mock <ISystemEnvironment>();
env32.SetupGet(s => s.Is32BitProcess)
.Returns(() => true);
var env64 = new Mock <ISystemEnvironment>();
env64.SetupGet(s => s.Is32BitProcess)
.Returns(() => false);
var provider32 = new RocksDbOptionsProvider(
env32.Object,
true,
Option.None <ulong>(),
Option.None <ulong>(),
Option.None <int>(),
Option.None <StorageLogLevel>());
var provider64 = new RocksDbOptionsProvider(
env64.Object,
true,
Option.None <ulong>(),
Option.None <ulong>(),
Option.None <int>(),
Option.None <StorageLogLevel>());
// act
ColumnFamilyOptions newOptions32 = provider32.GetColumnFamilyOptions();
ColumnFamilyOptions newOptions64 = provider64.GetColumnFamilyOptions();
// assert
Assert.NotNull(newOptions32);
Assert.NotNull(newOptions64);
}
public ColumnFamilyHandle CreateColumnFamily(ColumnFamilyOptions columnFamilyOptions, string entityName)
{
lock (ColumnFamiliesLock)
{
this.columnFamiliesProvider.AddColumnFamily(entityName);
ColumnFamilyHandle handle = this.db.CreateColumnFamily(columnFamilyOptions, entityName);
return(handle);
}
}
/// <summary>
/// Creates a new column family named as the given <paramref name="name"/>,
/// on <see cref="RocksDbSharp.RocksDb"/> located at <paramref name="rocksdbPath"/>.
/// </summary>
/// <param name="name">The name of the column family to create.</param>
/// <param name="rocksdbPath">The path of <see cref="RocksDbSharp.RocksDb"/> to load.</param>
public void Create([Argument] string name, [Option] string? rocksdbPath = null)
{
rocksdbPath ??= Directory.GetCurrentDirectory();
using var db = _rocksDbService.Load(rocksdbPath);
var options = new ColumnFamilyOptions();
db.CreateColumnFamily(
options,
name);
}
private ColumnFamilyHandle CreateColumnFamilyIfMissing(RocksDb db, string columnFamily)
{
try
{
return(db.GetColumnFamily(columnFamily));
}
catch
{
var options = new ColumnFamilyOptions();
return(db.CreateColumnFamily(options, columnFamily));
}
}
private static ColumnFamilies GetColumnFamilies(IDbConfig dbConfig, string name, T[] keys)
{
var result = new ColumnFamilies();
var blockCacheSize = ReadConfig <ulong>(dbConfig, nameof(dbConfig.BlockCacheSize), name);
foreach (var key in keys)
{
var columnFamilyOptions = new ColumnFamilyOptions();
columnFamilyOptions.OptimizeForPointLookup(blockCacheSize);
columnFamilyOptions.SetBlockBasedTableFactory(new BlockBasedTableOptions().SetFilterPolicy(BloomFilterPolicy.Create()));
result.Add(key.ToString(), columnFamilyOptions);
}
return(result);
}
public ColumnFamilyOptions GetColumnFamilyOptions()
{
var options = new ColumnFamilyOptions();
if (this.env.Is32BitProcess)
{
// restrict some sizes if 32 bit OS.
options.SetWriteBufferSize(WriteBufferSize);
options.SetTargetFileSizeBase(TargetFileSizeBase);
options.SetMaxBytesForLevelBase(MaxBytesForLevelBase);
options.SetSoftPendingCompactionBytesLimit(SoftPendingCompactionBytes);
options.SetHardPendingCompactionBytesLimit(HardPendingCompactionBytes);
}
return(options);
}
static ColumnFamilies GetColumnFamilies(string path)
{
if (RocksDb.TryListColumnFamilies(new DbOptions(), path, out var names))
{
var columnFamilyOptions = new ColumnFamilyOptions();
var families = new ColumnFamilies();
foreach (var name in names)
{
families.Add(name, columnFamilyOptions);
}
return(families);
}
return(new ColumnFamilies());
}
private static void RocksDb2()
{
string directory = @"C:\Users\Peska\source\repos\Esent.Tests\RocksDB";
DbOptions options = new DbOptions().SetCreateIfMissing();
using (RocksDb rocksDb = RocksDbSharp.RocksDb.Open(options, directory))
{
TestObject o1 = new TestObject {
EQNum = "3L1234", Mnemonic = "20013L1234011"
};
TestObject o2 = new TestObject {
EQNum = "3L5678", Mnemonic = "20023L5678011"
};
TestObject o3 = new TestObject {
EQNum = "3L9012", Mnemonic = "20033L9012011"
};
TestObject o4 = new TestObject {
EQNum = "3L9012", Mnemonic = "20013L9012012"
};
rocksDb.Put(o1.Mnemonic, JsonConvert.SerializeObject(o1));
rocksDb.Put(o2.Mnemonic, JsonConvert.SerializeObject(o2));
rocksDb.Put(o3.Mnemonic, JsonConvert.SerializeObject(o3));
rocksDb.Put(o4.Mnemonic, JsonConvert.SerializeObject(o4));
SliceTransform sliceTransform = SliceTransform.CreateFixedPrefix(4);
BlockBasedTableOptions blockBasedTableOptions = new BlockBasedTableOptions().SetWholeKeyFiltering(false);
ColumnFamilyOptions columnFamilyOptions = new ColumnFamilyOptions()
.SetPrefixExtractor(sliceTransform)
.SetBlockBasedTableFactory(blockBasedTableOptions);
ColumnFamilies columnFamilies = new ColumnFamilies(columnFamilyOptions);
Iterator iterator = rocksDb.NewIterator();
iterator = iterator.Seek("2001");
while (iterator.Valid())
{
string key = iterator.StringKey();
string value = iterator.StringValue();
iterator.Next();
}
}
}
/// <summary>
/// Create rocksdb config and open rocksdb database.
/// </summary>
/// <param name="context"></param>
protected void OpenDatabase(ProcessorContext context)
{
DbOptions dbOptions = new DbOptions();
ColumnFamilyOptions columnFamilyOptions = new ColumnFamilyOptions();
writeOptions = new WriteOptions();
BlockBasedTableOptions tableConfig = new BlockBasedTableOptions();
RocksDbOptions rocksDbOptions = new RocksDbOptions(dbOptions, columnFamilyOptions);
tableConfig.SetBlockCache(RocksDbSharp.Cache.CreateLru(BLOCK_CACHE_SIZE));
tableConfig.SetBlockSize(BLOCK_SIZE);
tableConfig.SetFilterPolicy(BloomFilterPolicy.Create());
rocksDbOptions.SetOptimizeFiltersForHits(1);
rocksDbOptions.SetBlockBasedTableFactory(tableConfig);
rocksDbOptions.SetCompression(COMPRESSION_TYPE);
rocksDbOptions.SetWriteBufferSize(WRITE_BUFFER_SIZE);
rocksDbOptions.SetCompactionStyle(COMPACTION_STYLE);
rocksDbOptions.SetMaxWriteBufferNumber(MAX_WRITE_BUFFERS);
rocksDbOptions.SetCreateIfMissing(true);
rocksDbOptions.SetErrorIfExists(false);
rocksDbOptions.SetInfoLogLevel(RocksLogLevel.ERROR);
// this is the recommended way to increase parallelism in RocksDb
// note that the current implementation of setIncreaseParallelism affects the number
// of compaction threads but not flush threads (the latter remains one). Also
// the parallelism value needs to be at least two because of the code in
// https://github.com/facebook/rocksdb/blob/62ad0a9b19f0be4cefa70b6b32876e764b7f3c11/util/options.cc#L580
// subtracts one from the value passed to determine the number of compaction threads
// (this could be a bug in the RocksDB code and their devs have been contacted).
rocksDbOptions.IncreaseParallelism(Math.Max(Environment.ProcessorCount, 2));
// TODO : wrap writeOptions in rocksDbOptions too
writeOptions.DisableWal(1);
context.Configuration.RocksDbConfigHandler?.Invoke(Name, rocksDbOptions);
rocksDbOptions.SetMinWriteBufferNumberToMerge(2);
DbDir = new DirectoryInfo(Path.Combine(context.StateDir, parentDir, Name));
Directory.CreateDirectory(DbDir.FullName);
OpenRocksDb(dbOptions, columnFamilyOptions);
IsOpen = true;
}
/// <summary>
/// Open rocksdb handle
/// </summary>
/// <param name="dbOptions">Rocksdb options</param>
/// <param name="columnFamilyOptions">Columnfamily options</param>
/// <exception cref="ProcessorStateException">throws if the rocksdb can't be open</exception>
private void OpenRocksDb(DbOptions dbOptions, ColumnFamilyOptions columnFamilyOptions)
{
int maxRetries = 5;
int i = 0;
bool open = false;
RocksDbException rocksDbException = null;
var columnFamilyDescriptors = new ColumnFamilies(columnFamilyOptions);
while (!open && i < maxRetries)
{
try
{
Db = RocksDbSharp.RocksDb.Open(
dbOptions,
DbDir.FullName,
columnFamilyDescriptors);
var columnFamilyHandle = Db.GetDefaultColumnFamily();
DbAdapter = new SingleColumnFamilyAdapter(
Name,
Db,
writeOptions,
KeyComparator,
columnFamilyHandle);
open = true;
}
catch (RocksDbException e)
{
++i;
rocksDbException = e;
}
}
if (!open)
{
throw new ProcessorStateException("Error opening store " + Name + " at location " + DbDir.ToString(), rocksDbException);
}
}
public void FunctionalTest()
{
string temp = Path.GetTempPath();
var testdb = Path.Combine(temp, "functional_test");
string path = Environment.ExpandEnvironmentVariables(testdb);
if (Directory.Exists(testdb))
{
Directory.Delete(testdb, true);
}
var options = new DbOptions()
.SetCreateIfMissing(true)
.EnableStatistics();
// Using standard open
using (var db = RocksDb.Open(options, path))
{
// With strings
string value = db.Get("key");
db.Put("key", "value");
Assert.Equal("value", db.Get("key"));
Assert.Null(db.Get("non-existent-key"));
db.Remove("key");
Assert.Null(db.Get("value"));
// With bytes
db.Put(Encoding.UTF8.GetBytes("key"), Encoding.UTF8.GetBytes("value"));
Assert.True(BinaryComparer.Default.Equals(Encoding.UTF8.GetBytes("value"), db.Get(Encoding.UTF8.GetBytes("key"))));
// non-existent kiey
Assert.Null(db.Get(new byte[] { 0, 1, 2 }));
db.Remove(Encoding.UTF8.GetBytes("key"));
Assert.Null(db.Get(Encoding.UTF8.GetBytes("key")));
db.Put(Encoding.UTF8.GetBytes("key"), new byte[] { 0, 1, 2, 3, 4, 5, 6, 7 });
// With buffers
var buffer = new byte[100];
long length = db.Get(Encoding.UTF8.GetBytes("key"), buffer, 0, buffer.Length);
Assert.Equal(8, length);
Assert.Equal(new byte[] { 0, 1, 2, 3, 4, 5, 6, 7 }, buffer.Take((int)length).ToList());
// Write batches
// With strings
using (WriteBatch batch = new WriteBatch()
.Put("one", "uno")
.Put("two", "deuce")
.Put("two", "dos")
.Put("three", "tres"))
{
db.Write(batch);
}
Assert.Equal("uno", db.Get("one"));
// With save point
using (WriteBatch batch = new WriteBatch())
{
batch
.Put("hearts", "red")
.Put("diamonds", "red");
batch.SetSavePoint();
batch
.Put("clubs", "black");
batch.SetSavePoint();
batch
.Put("spades", "black");
batch.RollbackToSavePoint();
db.Write(batch);
}
Assert.Equal("red", db.Get("diamonds"));
Assert.Equal("black", db.Get("clubs"));
Assert.Null(db.Get("spades"));
// With bytes
var utf8 = Encoding.UTF8;
using (WriteBatch batch = new WriteBatch()
.Put(utf8.GetBytes("four"), new byte[] { 4, 4, 4 })
.Put(utf8.GetBytes("five"), new byte[] { 5, 5, 5 }))
{
db.Write(batch);
}
Assert.True(BinaryComparer.Default.Equals(new byte[] { 4, 4, 4 }, db.Get(utf8.GetBytes("four"))));
// Snapshots
using (var snapshot = db.CreateSnapshot())
{
var before = db.Get("one");
db.Put("one", "1");
var useSnapshot = new ReadOptions()
.SetSnapshot(snapshot);
// the database value was written
Assert.Equal("1", db.Get("one"));
// but the snapshot still sees the old version
var after = db.Get("one", readOptions: useSnapshot);
Assert.Equal(before, after);
}
var two = db.Get("two");
Assert.Equal("dos", two);
// Iterators
using (var iterator = db.NewIterator(
readOptions: new ReadOptions()
.SetIterateUpperBound("t")
))
{
iterator.Seek("k");
Assert.True(iterator.Valid());
Assert.Equal("key", iterator.StringKey());
iterator.Next();
Assert.True(iterator.Valid());
Assert.Equal("one", iterator.StringKey());
Assert.Equal("1", iterator.StringValue());
iterator.Next();
Assert.False(iterator.Valid());
}
// MultiGet
var multiGetResult = db.MultiGet(new[] { "two", "three", "nine" });
Assert.Equal(
expected: new[]
{
new KeyValuePair <string, string>("two", "dos"),
new KeyValuePair <string, string>("three", "tres"),
new KeyValuePair <string, string>("nine", null)
},
actual: multiGetResult
);
}
// Test with column families
var optionsCf = new DbOptions()
.SetCreateIfMissing(true)
.SetCreateMissingColumnFamilies(true);
var columnFamilies = new ColumnFamilies
{
{ "reverse", new ColumnFamilyOptions() },
};
using (var db = RocksDb.Open(optionsCf, path, columnFamilies))
{
var reverse = db.GetColumnFamily("reverse");
db.Put("one", "uno");
db.Put("two", "dos");
db.Put("three", "tres");
db.Put("uno", "one", cf: reverse);
db.Put("dos", "two", cf: reverse);
db.Put("tres", "three", cf: reverse);
}
// Test Cf Delete
using (var db = RocksDb.Open(optionsCf, path, columnFamilies))
{
var reverse = db.GetColumnFamily("reverse");
db.Put("cuatro", "four", cf: reverse);
db.Put("cinco", "five", cf: reverse);
Assert.Equal("four", db.Get("cuatro", cf: reverse));
Assert.Equal("five", db.Get("cinco", cf: reverse));
byte[] keyBytes = Encoding.UTF8.GetBytes("cuatro");
db.Remove(keyBytes, reverse);
db.Remove("cinco", reverse);
Assert.Null(db.Get("cuatro", cf: reverse));
Assert.Null(db.Get("cinco", cf: reverse));
}
// Test list
{
var list = RocksDb.ListColumnFamilies(optionsCf, path);
Assert.Equal(new[] { "default", "reverse" }, list.ToArray());
}
// Test reopen with column families
using (var db = RocksDb.Open(optionsCf, path, columnFamilies))
{
var reverse = db.GetColumnFamily("reverse");
Assert.Equal("uno", db.Get("one"));
Assert.Equal("one", db.Get("uno", cf: reverse));
Assert.Null(db.Get("uno"));
Assert.Null(db.Get("one", cf: reverse));
}
// Test dropping and creating column family
using (var db = RocksDb.Open(options, path, columnFamilies))
{
db.DropColumnFamily("reverse");
var reverse = db.CreateColumnFamily(new ColumnFamilyOptions(), "reverse");
Assert.Null(db.Get("uno", cf: reverse));
db.Put("red", "rouge", cf: reverse);
Assert.Equal("rouge", db.Get("red", cf: reverse));
}
// Test reopen after drop and create
using (var db = RocksDb.Open(options, path, columnFamilies))
{
var reverse = db.GetColumnFamily("reverse");
Assert.Null(db.Get("uno", cf: reverse));
Assert.Equal("rouge", db.Get("red", cf: reverse));
}
// Test read only
using (var db = RocksDb.OpenReadOnly(options, path, columnFamilies, false))
{
Assert.Equal("uno", db.Get("one"));
}
// Test SstFileWriter
{
var envOpts = new EnvOptions();
var ioOpts = new ColumnFamilyOptions();
var sst = new SstFileWriter(envOpts, ioOpts);
var filename = Path.Combine(temp, "test.sst");
if (File.Exists(filename))
{
File.Delete(filename);
}
sst.Open(filename);
sst.Add("four", "quatro");
sst.Add("one", "uno");
sst.Add("two", "dos");
sst.Finish();
using (var db = RocksDb.Open(options, path, columnFamilies))
{
Assert.NotEqual("four", db.Get("four"));
var ingestOptions = new IngestExternalFileOptions()
.SetMoveFiles(true);
db.IngestExternalFiles(new string[] { filename }, ingestOptions);
Assert.Equal("quatro", db.Get("four"));
}
}
// test comparator
unsafe {
var comparator = new IntegerStringComparator();
var opts = new ColumnFamilyOptions()
.SetComparator(comparator);
var filename = Path.Combine(temp, "test.sst");
if (File.Exists(filename))
{
File.Delete(filename);
}
var sst = new SstFileWriter(ioOptions: opts);
sst.Open(filename);
sst.Add("111", "111");
sst.Add("1001", "1001"); // this order is only allowed using an integer comparator
sst.Finish();
}
// test write batch with index
{
var wbwi = new WriteBatchWithIndex(reservedBytes: 1024);
wbwi.Put("one", "un");
wbwi.Put("two", "deux");
var oneValueIn = Encoding.UTF8.GetBytes("one");
var oneValueOut = wbwi.Get("one");
Assert.Equal("un", oneValueOut);
using (var db = RocksDb.Open(options, path, columnFamilies))
{
var oneCombinedOut = wbwi.Get(db, "one");
var threeCombinedOut = wbwi.Get(db, "three");
Assert.Equal("un", oneCombinedOut);
Assert.Equal("tres", threeCombinedOut);
using (var wbIterator = wbwi.NewIterator(db.NewIterator()))
{
wbIterator.Seek("o");
Assert.True(wbIterator.Valid());
var itkey = wbIterator.StringKey();
Assert.Equal("one", itkey);
var itval = wbIterator.StringValue();
Assert.Equal("un", itval);
wbIterator.Next();
Assert.True(wbIterator.Valid());
itkey = wbIterator.StringKey();
Assert.Equal("three", itkey);
itval = wbIterator.StringValue();
Assert.Equal("tres", itval);
wbIterator.Next();
Assert.True(wbIterator.Valid());
itkey = wbIterator.StringKey();
Assert.Equal("two", itkey);
itval = wbIterator.StringValue();
Assert.Equal("deux", itval);
wbIterator.Next();
Assert.False(wbIterator.Valid());
}
db.Write(wbwi);
var oneDbOut = wbwi.Get("one");
Assert.Equal("un", oneDbOut);
}
}
// compact range
{
using (var db = RocksDb.Open(options, path, columnFamilies))
{
db.CompactRange("o", "tw");
}
}
}
/// <summary>
/// Initializes a new instance of the <see cref="RocksDbService"/> class.
/// </summary>
public RocksDbService()
{
_options = new DbOptions();
_columnFamilyOptions = new ColumnFamilyOptions();
}
public void CreateCF(string name)
{
var cf = new ColumnFamilyOptions();
_db.CreateColumnFamily(cf, name);
}
public ColumnFamilyHandle CreateColumnFamily(ColumnFamilyOptions columnFamilyOptions, string entityName) => this.db.CreateColumnFamily(columnFamilyOptions, entityName);
public void FunctionalTest()
{
string temp = Path.GetTempPath();
var testdir = Path.Combine(temp, "functional_test");
var testdb = Path.Combine(testdir, "main");
var testcp = Path.Combine(testdir, "cp");
var path = Environment.ExpandEnvironmentVariables(testdb);
var cppath = Environment.ExpandEnvironmentVariables(testcp);
if (Directory.Exists(testdir))
{
Directory.Delete(testdir, true);
}
Directory.CreateDirectory(testdir);
var options = new DbOptions()
.SetCreateIfMissing(true)
.EnableStatistics();
// Using standard open
using (var db = RocksDb.Open(options, path))
{
// With strings
string value = db.Get("key");
db.Put("key", "value");
Assert.Equal("value", db.Get("key"));
Assert.Null(db.Get("non-existent-key"));
db.Remove("key");
Assert.Null(db.Get("value"));
// With bytes
db.Put(Encoding.UTF8.GetBytes("key"), Encoding.UTF8.GetBytes("value"));
Assert.True(BinaryComparer.Default.Equals(Encoding.UTF8.GetBytes("value"), db.Get(Encoding.UTF8.GetBytes("key"))));
// non-existent kiey
Assert.Null(db.Get(new byte[] { 0, 1, 2 }));
db.Remove(Encoding.UTF8.GetBytes("key"));
Assert.Null(db.Get(Encoding.UTF8.GetBytes("key")));
db.Put(Encoding.UTF8.GetBytes("key"), new byte[] { 0, 1, 2, 3, 4, 5, 6, 7 });
// With buffers
var buffer = new byte[100];
long length = db.Get(Encoding.UTF8.GetBytes("key"), buffer, 0, buffer.Length);
Assert.Equal(8, length);
Assert.Equal(new byte[] { 0, 1, 2, 3, 4, 5, 6, 7 }, buffer.Take((int)length).ToList());
buffer = new byte[5];
length = db.Get(Encoding.UTF8.GetBytes("key"), buffer, 0, buffer.Length);
Assert.Equal(8, length);
Assert.Equal(new byte[] { 0, 1, 2, 3, 4 }, buffer.Take((int)Math.Min(buffer.Length, length)));
length = db.Get(Encoding.UTF8.GetBytes("bogus"), buffer, 0, buffer.Length);
Assert.Equal(-1, length);
// Write batches
// With strings
using (WriteBatch batch = new WriteBatch()
.Put("one", "uno")
.Put("two", "deuce")
.Put("two", "dos")
.Put("three", "tres"))
{
db.Write(batch);
}
Assert.Equal("uno", db.Get("one"));
// With save point
using (WriteBatch batch = new WriteBatch())
{
batch
.Put("hearts", "red")
.Put("diamonds", "red");
batch.SetSavePoint();
batch
.Put("clubs", "black");
batch.SetSavePoint();
batch
.Put("spades", "black");
batch.RollbackToSavePoint();
db.Write(batch);
}
Assert.Equal("red", db.Get("diamonds"));
Assert.Equal("black", db.Get("clubs"));
Assert.Null(db.Get("spades"));
// Save a checkpoint
using (var cp = db.Checkpoint())
{
cp.Save(cppath);
}
// With bytes
var utf8 = Encoding.UTF8;
using (WriteBatch batch = new WriteBatch()
.Put(utf8.GetBytes("four"), new byte[] { 4, 4, 4 })
.Put(utf8.GetBytes("five"), new byte[] { 5, 5, 5 }))
{
db.Write(batch);
}
Assert.True(BinaryComparer.Default.Equals(new byte[] { 4, 4, 4 }, db.Get(utf8.GetBytes("four"))));
// Snapshots
using (var snapshot = db.CreateSnapshot())
{
var before = db.Get("one");
db.Put("one", "1");
var useSnapshot = new ReadOptions()
.SetSnapshot(snapshot);
// the database value was written
Assert.Equal("1", db.Get("one"));
// but the snapshot still sees the old version
var after = db.Get("one", readOptions: useSnapshot);
Assert.Equal(before, after);
}
var two = db.Get("two");
Assert.Equal("dos", two);
// Iterators
using (var iterator = db.NewIterator(
readOptions: new ReadOptions()
.SetIterateUpperBound("t")
))
{
iterator.Seek("k");
Assert.True(iterator.Valid());
Assert.Equal("key", iterator.StringKey());
iterator.Next();
Assert.True(iterator.Valid());
Assert.Equal("one", iterator.StringKey());
Assert.Equal("1", iterator.StringValue());
iterator.Next();
Assert.False(iterator.Valid());
}
// MultiGet
var multiGetResult = db.MultiGet(new[] { "two", "three", "nine" });
Assert.Equal(
expected: new[]
{
new KeyValuePair <string, string>("two", "dos"),
new KeyValuePair <string, string>("three", "tres"),
new KeyValuePair <string, string>("nine", null)
},
actual: multiGetResult
);
}
// Test reading checkpointed db
using (var cpdb = RocksDb.Open(options, cppath))
{
Assert.Equal("red", cpdb.Get("diamonds"));
Assert.Equal("black", cpdb.Get("clubs"));
Assert.Null(cpdb.Get("spades"));
// Checkpoint occurred before these changes:
Assert.Null(cpdb.Get("four"));
}
// Test with column families
var optionsCf = new DbOptions()
.SetCreateIfMissing(true)
.SetCreateMissingColumnFamilies(true);
var columnFamilies = new ColumnFamilies
{
{ "reverse", new ColumnFamilyOptions() },
};
using (var db = RocksDb.Open(optionsCf, path, columnFamilies))
{
var reverse = db.GetColumnFamily("reverse");
db.Put("one", "uno");
db.Put("two", "dos");
db.Put("three", "tres");
db.Put("uno", "one", cf: reverse);
db.Put("dos", "two", cf: reverse);
db.Put("tres", "three", cf: reverse);
}
// Test Cf Delete
using (var db = RocksDb.Open(optionsCf, path, columnFamilies))
{
var reverse = db.GetColumnFamily("reverse");
db.Put("cuatro", "four", cf: reverse);
db.Put("cinco", "five", cf: reverse);
Assert.Equal("four", db.Get("cuatro", cf: reverse));
Assert.Equal("five", db.Get("cinco", cf: reverse));
byte[] keyBytes = Encoding.UTF8.GetBytes("cuatro");
db.Remove(keyBytes, reverse);
db.Remove("cinco", reverse);
Assert.Null(db.Get("cuatro", cf: reverse));
Assert.Null(db.Get("cinco", cf: reverse));
}
// Test list
{
var list = RocksDb.ListColumnFamilies(optionsCf, path);
Assert.Equal(new[] { "default", "reverse" }, list.ToArray());
}
// Test reopen with column families
using (var db = RocksDb.Open(optionsCf, path, columnFamilies))
{
var reverse = db.GetColumnFamily("reverse");
Assert.Equal("uno", db.Get("one"));
Assert.Equal("one", db.Get("uno", cf: reverse));
Assert.Null(db.Get("uno"));
Assert.Null(db.Get("one", cf: reverse));
}
// Test dropping and creating column family
using (var db = RocksDb.Open(options, path, columnFamilies))
{
db.DropColumnFamily("reverse");
var reverse = db.CreateColumnFamily(new ColumnFamilyOptions(), "reverse");
Assert.Null(db.Get("uno", cf: reverse));
db.Put("red", "rouge", cf: reverse);
Assert.Equal("rouge", db.Get("red", cf: reverse));
}
// Test reopen after drop and create
using (var db = RocksDb.Open(options, path, columnFamilies))
{
var reverse = db.GetColumnFamily("reverse");
Assert.Null(db.Get("uno", cf: reverse));
Assert.Equal("rouge", db.Get("red", cf: reverse));
}
// Test read only
using (var db = RocksDb.OpenReadOnly(options, path, columnFamilies, false))
{
Assert.Equal("uno", db.Get("one"));
}
// Test SstFileWriter
{
using (var writer = new SstFileWriter())
{
}
var envOpts = new EnvOptions();
var ioOpts = new ColumnFamilyOptions();
using (var sst = new SstFileWriter(envOpts, ioOpts))
{
var filename = Path.Combine(temp, "test.sst");
if (File.Exists(filename))
{
File.Delete(filename);
}
sst.Open(filename);
sst.Add("four", "quatro");
sst.Add("one", "uno");
sst.Add("two", "dos");
sst.Finish();
using (var db = RocksDb.Open(options, path, columnFamilies))
{
Assert.NotEqual("four", db.Get("four"));
var ingestOptions = new IngestExternalFileOptions()
.SetMoveFiles(true);
db.IngestExternalFiles(new string[] { filename }, ingestOptions);
Assert.Equal("quatro", db.Get("four"));
}
}
}
// test comparator
unsafe {
var opts = new ColumnFamilyOptions()
.SetComparator(new IntegerStringComparator());
var filename = Path.Combine(temp, "test.sst");
if (File.Exists(filename))
{
File.Delete(filename);
}
using (var sst = new SstFileWriter(ioOptions: opts))
{
sst.Open(filename);
sst.Add("111", "111");
sst.Add("1001", "1001"); // this order is only allowed using an integer comparator
sst.Finish();
}
}
// test write batch with index
{
var wbwi = new WriteBatchWithIndex(reservedBytes: 1024);
wbwi.Put("one", "un");
wbwi.Put("two", "deux");
var oneValueIn = Encoding.UTF8.GetBytes("one");
var oneValueOut = wbwi.Get("one");
Assert.Equal("un", oneValueOut);
using (var db = RocksDb.Open(options, path, columnFamilies))
{
var oneCombinedOut = wbwi.Get(db, "one");
var threeCombinedOut = wbwi.Get(db, "three");
Assert.Equal("un", oneCombinedOut);
Assert.Equal("tres", threeCombinedOut);
using (var wbIterator = wbwi.NewIterator(db.NewIterator()))
{
wbIterator.Seek("o");
Assert.True(wbIterator.Valid());
var itkey = wbIterator.StringKey();
Assert.Equal("one", itkey);
var itval = wbIterator.StringValue();
Assert.Equal("un", itval);
wbIterator.Next();
Assert.True(wbIterator.Valid());
itkey = wbIterator.StringKey();
Assert.Equal("three", itkey);
itval = wbIterator.StringValue();
Assert.Equal("tres", itval);
wbIterator.Next();
Assert.True(wbIterator.Valid());
itkey = wbIterator.StringKey();
Assert.Equal("two", itkey);
itval = wbIterator.StringValue();
Assert.Equal("deux", itval);
wbIterator.Next();
Assert.False(wbIterator.Valid());
}
db.Write(wbwi);
var oneDbOut = wbwi.Get("one");
Assert.Equal("un", oneDbOut);
}
}
// compact range
{
using (var db = RocksDb.Open(options, path, columnFamilies))
{
db.CompactRange("o", "tw");
}
}
// Smoke test various options
{
var dbname = "test-options";
if (Directory.Exists(dbname))
{
Directory.Delete(dbname, true);
}
var optsTest = (DbOptions) new RocksDbSharp.DbOptions()
.SetCreateIfMissing(true)
.SetCreateMissingColumnFamilies(true)
.SetBlockBasedTableFactory(new BlockBasedTableOptions().SetBlockCache(Cache.CreateLru(1024 * 1024)));
GC.Collect();
using (var db = RocksDbSharp.RocksDb.Open(optsTest, dbname))
{
}
if (Directory.Exists(dbname))
{
Directory.Delete(dbname, true);
}
}
// Smoke test OpenWithTtl
{
var dbname = "test-with-ttl";
if (Directory.Exists(dbname))
{
Directory.Delete(dbname, true);
}
var optsTest = (DbOptions) new RocksDbSharp.DbOptions()
.SetCreateIfMissing(true)
.SetCreateMissingColumnFamilies(true);
using (var db = RocksDbSharp.RocksDb.OpenWithTtl(optsTest, dbname, 1))
{
}
if (Directory.Exists(dbname))
{
Directory.Delete(dbname, true);
}
}
// Smoke test MergeOperator
{
var dbname = "test-merge-operator";
if (Directory.Exists(dbname))
{
Directory.Delete(dbname, true);
}
var optsTest = (DbOptions) new RocksDbSharp.DbOptions()
.SetCreateIfMissing(true)
.SetMergeOperator(MergeOperators.Create(
name: "test-merge-operator",
partialMerge: (key, keyLength, operandsList, operandsListLength, numOperands, success, newValueLength) => IntPtr.Zero,
fullMerge: (key, keyLength, existingValue, existingValueLength, operandsList, operandsListLength, numOperands, success, newValueLength) => IntPtr.Zero,
deleteValue: (value, valueLength) => { }
));
GC.Collect();
using (var db = RocksDbSharp.RocksDb.Open(optsTest, dbname))
{
}
if (Directory.Exists(dbname))
{
Directory.Delete(dbname, true);
}
}
// Test that GC does not cause access violation on Comparers
{
var dbname = "test-av-error";
if (Directory.Exists(dbname))
{
Directory.Delete(dbname, true);
}
options = new RocksDbSharp.DbOptions()
.SetCreateIfMissing(true)
.SetCreateMissingColumnFamilies(true);
var sc = new RocksDbSharp.StringComparator(StringComparer.InvariantCultureIgnoreCase);
columnFamilies = new RocksDbSharp.ColumnFamilies
{
{ "cf1", new RocksDbSharp.ColumnFamilyOptions()
.SetComparator(sc) },
};
GC.Collect();
using (var db = RocksDbSharp.RocksDb.Open(options, dbname, columnFamilies))
{
}
if (Directory.Exists(dbname))
{
Directory.Delete(dbname, true);
}
}
}
