public async static Task PeriodicallyDisplayTopCountsState(string brokerList, RocksDb db, CancellationToken ct)
{
while (true)
{
await Task.Delay(10000, ct);
var it = db.NewIterator(db.GetColumnFamily("counts")).SeekToFirst();
var N = 5;
var maxWords = new List<(int, string)>();
while (it.Valid())
{
var wc = (BitConverter.ToInt32(it.Value()), Encoding.UTF8.GetString(it.Key()));
if (maxWords.Count < N) { maxWords.Add(wc); }
else { if (wc.Item1 > maxWords[N-1].Item1) { maxWords[N-1] = wc; } }
maxWords.Sort((x, y) => y.Item1.CompareTo(x.Item1));
it.Next();
}
if (maxWords.Count > 0) { Console.WriteLine("Most frequently occuring words known to this instance:"); }
foreach (var wc in maxWords)
{
Console.WriteLine(" " + wc.Item2 + " " + wc.Item1);
}
}
}
public bool MoveNext()
{
if (_currentKey == null)
{
_currentKey = _start;
// todo: check
_iterator = _db.NewIterator();
_iterator.Seek(_start);
var firstKey = _iterator.Key();
if (KeyConfig.ByteCompare(firstKey, _start) < 0)
{
return(false);
}
}
else
{
_nextFunction(_iterator);
_currentKey = _iterator.Key();
}
if (!_iterator.Valid())
{
return(false);
}
var key = _iterator.Key();
if (KeyConfig.ByteCompare(key, _end) > 0)
{
return(false);
}
return(true);
}
public static IEnumerable <(byte[] key, byte[] value)> Seek(this RocksDb db, ColumnFamilyHandle columnFamily, ReadOnlySpan <byte> prefix, SeekDirection direction, ReadOptions?readOptions)
{
var iterator = db.NewIterator(columnFamily, readOptions);
if (direction == SeekDirection.Forward)
{
iterator.Seek(prefix);
return(SeekInternal(iterator, iterator.Next));
}
else
{
iterator.SeekForPrev(prefix);
return(SeekInternal(iterator, iterator.Prev));
}
IEnumerable <(byte[] key, byte[] value)> SeekInternal(Iterator iterator, Func <Iterator> nextAction)
{
using (iterator)
{
while (iterator.Valid())
{
yield return(iterator.Key(), iterator.Value());
nextAction();
}
}
}
}
public static bool ColumnFamilyEmpty(this RocksDb db, ColumnFamilyHandle columnFamily, ReadOptions?readOptions = null)
{
var iter = db.NewIterator(columnFamily, readOptions);
iter.SeekToFirst();
return(!iter.Valid());
}
public IEnumerable <(byte[] Key, byte[] Value)> Seek(byte[] keyOrPrefix, SeekDirection direction)
{
if (keyOrPrefix == null)
{
keyOrPrefix = Array.Empty <byte>();
}
using var it = db.NewIterator(store.defaultFamily, options);
byte[] fullKey = keyOrPrefix;
if (direction == SeekDirection.Forward)
{
for (it.Seek(fullKey); it.Valid(); it.Next())
{
yield return(it.Key(), it.Value());
}
}
else
{
for (it.SeekForPrev(fullKey); it.Valid(); it.Prev())
{
yield return(it.Key(), it.Value());
}
}
}
public async Task FlushCacheAsync(CancellationToken cancellationToken)
{
if (_bypassLocalCache)
{
throw new NotSupportedException();
}
var writeBatch = new List <ContribSampleEntity>();
var currentTime = DateTime.UtcNow;
var cutoffTime = new DateTime(currentTime.Year, currentTime.Month,
currentTime.Day, currentTime.Hour, 0, 0, DateTimeKind.Utc);
var zeroStream = new MemoryStream();
_formatter.Serialize(zeroStream, 0);
var zeroBuffer = zeroStream.GetBuffer();
using (var i = _preAggDatabase.NewIterator())
{
while (i.Valid())
{
var k = Encoding.UTF8.GetString(i.Key());
var v = (long)_formatter.Deserialize(new MemoryStream(i.Value()));
writeBatch.Add(new ContribSampleEntity(k, cutoffTime, v));
_preAggDatabase.Put(i.Key(), zeroBuffer);
i.Next();
}
}
await IngestSamplesToXTableAsync(writeBatch, cancellationToken);
}
public uint GetCount()
{
uint count = 0;
var readOptions = new ReadOptions();
using (var iter = _db.NewIterator(readOptions: readOptions, cf: GetPartition()))
{
iter.SeekToFirst();
while (iter.Valid())
{
count++;
iter.Next();
}
}
return(count);
}
public IEnumerable<(byte[] Key, byte[] Value)> Seek(byte table, byte[] keyOrPrefix, SeekDirection direction)
{
if (keyOrPrefix == null) keyOrPrefix = Array.Empty<byte>();
using var it = db.NewIterator(store.GetFamily(table), options);
if (direction == SeekDirection.Forward)
for (it.Seek(keyOrPrefix); it.Valid(); it.Next())
yield return (it.Key(), it.Value());
private static void RocksDb1()
{
string directory = @"C:\Users\Peska\source\repos\Esent.Tests\RocksDB";
int maxSize = 1_000_000;
TestObject[] objects = new TestObject[maxSize];
for (int i = 0; i < maxSize; i++)
{
objects[i] = TestObject.Create();
if (i % 1000 == 0)
{
Console.WriteLine($"Created {i} records");
}
}
DbOptions options = new DbOptions().SetCreateIfMissing();
using (RocksDb rocksDb = RocksDbSharp.RocksDb.Open(options, directory))
{
for (int i = 0; i < maxSize; i++)
{
string json = JsonConvert.SerializeObject(objects[i]);
rocksDb.Put(objects[i].EQNum, json);
if (i % 1000 == 0)
{
Console.WriteLine($"Indexed {i} records");
}
}
Random random = new Random();
for (int i = 0; i < maxSize; i++)
{
int randomInt = random.Next(maxSize - 1);
TestObject testObject = objects[randomInt];
TestObject fromDict = JsonConvert.DeserializeObject <TestObject>(rocksDb.Get(testObject.EQNum));
if (testObject.EQNum != fromDict.EQNum)
{
throw new Exception("What the f**k");
}
if (i % 1000 == 0)
{
Console.WriteLine($"Retrieved {i} records");
}
}
Iterator iterator = rocksDb.NewIterator();
}
}
private IEnumerable <Iterator> IterateDb(RocksDb db, byte[] prefix, Guid?chainId = null)
{
ColumnFamilyHandle cf = GetColumnFamily(db, chainId);
using Iterator it = db.NewIterator(cf);
for (it.Seek(prefix); it.Valid() && it.Key().StartsWith(prefix); it.Next())
{
yield return(it);
}
}
public IEnumerable <(byte[] Key, byte[] Value)> Seek(byte table, byte[] keyOrPrefix, SeekDirection direction = SeekDirection.Forward)
{
if (keyOrPrefix == null)
{
keyOrPrefix = Array.Empty <byte>();
}
byte[] fullKey = getFullKey(table, keyOrPrefix);
using var it = db.NewIterator(defaultFamily, Options.ReadDefault);
if (direction == SeekDirection.Forward)
{
for (it.Seek(fullKey); it.Valid() && it.Key()[0] == table; it.Next())
{
yield return(it.Key()[1..], it.Value());
private T WithIterator <T>(Func <Iterator, T> body)
{
var ro = RocksDbSnapshot();
var iterator = database.NewIterator(cf: null, readOptions: ro);
try
{
return(body(iterator));
}
finally
{
iterator.Dispose();
}
}
public static IEnumerable <KeyValuePair <byte[], TValue> > Find <TValue>(this RocksDb db, byte[] keyPrefix, ColumnFamilyHandle?columnFamily = null, ReadOptions?readOptions = null)
where TValue : ISerializable, new()
{
using (var iterator = db.NewIterator(columnFamily, readOptions))
{
iterator.Seek(keyPrefix);
while (iterator.Valid())
{
yield return(new KeyValuePair <byte[], TValue>(
iterator.Key(), iterator.Value().AsSerializable <TValue>()));
iterator.Next();
}
}
}
public static Dictionary <byte[], byte[]> SelectDictionary(this RocksDb db, byte table)
{
var dict = new Dictionary <byte[], byte[]>();
var enumerator = db.NewIterator();
for (enumerator.SeekToFirst(); enumerator.Valid(); enumerator.Next())
{
if (enumerator.Key()[0] == table)
{
dict.Add(enumerator.Key().AsSpan().Slice(1).ToArray(), enumerator.Value());
}
}
return(dict);
}
private static void RocksDb2()
{
string directory = @"C:\Users\Peska\source\repos\Esent.Tests\RocksDB";
DbOptions options = new DbOptions().SetCreateIfMissing();
using (RocksDb rocksDb = RocksDbSharp.RocksDb.Open(options, directory))
{
TestObject o1 = new TestObject {
EQNum = "3L1234", Mnemonic = "20013L1234011"
};
TestObject o2 = new TestObject {
EQNum = "3L5678", Mnemonic = "20023L5678011"
};
TestObject o3 = new TestObject {
EQNum = "3L9012", Mnemonic = "20033L9012011"
};
TestObject o4 = new TestObject {
EQNum = "3L9012", Mnemonic = "20013L9012012"
};
rocksDb.Put(o1.Mnemonic, JsonConvert.SerializeObject(o1));
rocksDb.Put(o2.Mnemonic, JsonConvert.SerializeObject(o2));
rocksDb.Put(o3.Mnemonic, JsonConvert.SerializeObject(o3));
rocksDb.Put(o4.Mnemonic, JsonConvert.SerializeObject(o4));
SliceTransform sliceTransform = SliceTransform.CreateFixedPrefix(4);
BlockBasedTableOptions blockBasedTableOptions = new BlockBasedTableOptions().SetWholeKeyFiltering(false);
ColumnFamilyOptions columnFamilyOptions = new ColumnFamilyOptions()
.SetPrefixExtractor(sliceTransform)
.SetBlockBasedTableFactory(blockBasedTableOptions);
ColumnFamilies columnFamilies = new ColumnFamilies(columnFamilyOptions);
Iterator iterator = rocksDb.NewIterator();
iterator = iterator.Seek("2001");
while (iterator.Valid())
{
string key = iterator.StringKey();
string value = iterator.StringValue();
iterator.Next();
}
}
}
void ReadPackets(RocksDb db, short[] apidCounts)
{
var readStopwatch = new Stopwatch();
TimeSpan slowestRead = new TimeSpan();
var slowestCount = 0L;
while (!isFinished)
{
for (var apid = 0; apid < apidCounts.Length; apid++)
{
if (isFinished)
{
break;
}
if (apidCounts[apid] > 0)
{
readStopwatch.Reset();
readStopwatch.Start();
var firstKey = KeyToByteArray(new PacketKey {
APID = (short)apid, Timestamp = 0L
});
var lastKey = KeyToByteArray(new PacketKey {
APID = (short)apid, Timestamp = long.MaxValue
});
var count = 0L;
var iterator = db.NewIterator();
for (iterator.Seek(firstKey); iterator.Valid(); iterator.Next())
{
if (!ByteArrayLessThanEqualTo(iterator.Key(), lastKey))
{
break;
}
count++;
}
readStopwatch.Stop();
if (readStopwatch.ElapsedTicks > slowestRead.Ticks)
{
slowestRead = readStopwatch.Elapsed;
slowestCount = count;
}
}
System.Threading.Thread.Sleep(10);
}
}
Console.WriteLine($"Read count={slowestCount} slowest read={slowestRead}");
}
public IEnumerable <(byte[] Key, byte[] Value)> Seek(byte table, byte[] prefix, SeekDirection direction = SeekDirection.Forward)
{
using var it = db.NewIterator(GetFamily(table), Options.ReadDefault);
for (it.Seek(prefix); it.Valid();)
{
yield return(it.Key(), it.Value());
if (direction == SeekDirection.Forward)
{
it.Next();
}
else
{
it.Prev();
}
}
}
public IEnumerable <(byte[] Key, byte[] Value)> Seek(byte table, byte[] keyOrPrefix, SeekDirection direction)
{
using var it = db.NewIterator(store.GetFamily(table), options);
for (it.Seek(keyOrPrefix); it.Valid();)
{
yield return(it.Key(), it.Value());
if (direction == SeekDirection.Forward)
{
it.Next();
}
else
{
it.Prev();
}
}
}
public byte[][] GetAll()
{
var iterator = _db.NewIterator();
iterator = iterator.SeekToFirst();
var values = new List <byte[]>();
while (iterator.Valid())
{
values.Add(iterator.Value());
iterator = iterator.Next();
}
iterator.Dispose();
return(values.ToArray());
}
private ICollection <byte[]> GetKeysOrValues(Func <Iterator, byte[]> selector)
{
ReadOptions readOptions = new ReadOptions();
List <byte[]> items = new List <byte[]>();
using (Iterator iter = _db.NewIterator(readOptions: readOptions))
{
iter.SeekToFirst();
while (iter.Valid())
{
byte[] item = selector.Invoke(iter);
items.Add(item);
iter.Next();
}
}
return(items);
}
public IEnumerable <(byte[] Key, byte[] Value)> Find(byte table, byte[] prefix)
{
using var it = db.NewIterator(store.GetFamily(table), options);
for (it.Seek(prefix); it.Valid(); it.Next())
{
var key = it.Key();
byte[] y = prefix;
if (key.Length < y.Length)
{
break;
}
if (!key.AsSpan().StartsWith(y))
{
break;
}
yield return(key, it.Value());
}
}
IEnumerable <StoredValue> GetForPrefix(RocksDb db, ColumnFamilyHandle cf)
{
var readOptions = new ReadOptions();
using (var iter = db.NewIterator(readOptions: readOptions, cf: cf))
{
GC.Collect();
GC.WaitForPendingFinalizers();
var b = Encoding.UTF8.GetBytes($"{_data.LocatorString}");
iter.Seek(b);
while (iter.Valid())
{
Console.WriteLine(iter.StringKey());
yield return(iter.StringValue().Parse <StoredValue>());
iter.Next();
}
}
}
public RocksDbStore(string path)
{
var options = new DbOptions()
.SetCreateIfMissing(true)
.SetCreateMissingColumnFamilies(true);
db = RocksDb.Open(options, path, ColumnFamilies);
var writeBatch = new WriteBatch();
var readOptions = new ReadOptions().SetFillCache(true);
using (Iterator it = db.NewIterator(readOptions: readOptions))
{
for (it.SeekToFirst(); it.Valid(); it.Next())
{
writeBatch.Delete(it.Key());
}
}
db.Write(writeBatch);
}
private static void RocksDb4()
{
string directory = @"C:\Users\Peska\source\repos\Esent.Tests\RocksDB";
DbOptions options = new DbOptions().SetCreateIfMissing(true);
//SliceTransform sliceTransform = SliceTransform.CreateFixedPrefix(3);
//ColumnFamilyOptions columnFamilyOptions = new ColumnFamilyOptions().SetPrefixExtractor(sliceTransform);
//ColumnFamilies columnFamilies = new ColumnFamilies(columnFamilyOptions);
using (RocksDb rocksDb = RocksDb.Open(options, directory))
{
rocksDb.Put("AABCE", "Test 3");
rocksDb.Put("AAAAB", "Test 1");
rocksDb.Put("AAABB", "Test 2");
rocksDb.Put("AABCF", "Test 3");
rocksDb.Put("AABBB", "Test 3");
rocksDb.Put("ABBBB", "Test 4");
rocksDb.Put("AABCC", "Test 3");
rocksDb.Put("BBBBB", "Test 5");
rocksDb.Put("AABCC", "Test 3");
List <string> keys = new List <string>();
using (Iterator iterator = rocksDb.NewIterator())
{
iterator.Seek("AAB");
while (iterator.Valid())
{
keys.Add(iterator.StringKey());
iterator.Next();
}
}
Console.WriteLine(string.Join(", ", keys));
Console.ReadLine();
}
}
public IEnumerable <(byte[] Key, byte[] Value)> Seek(byte table, byte[] keyOrPrefix, SeekDirection direction = SeekDirection.Forward)
{
if (keyOrPrefix == null)
{
keyOrPrefix = Array.Empty <byte>();
}
using var it = db.NewIterator(GetFamily(table), Options.ReadDefault);
if (direction == SeekDirection.Forward)
{
for (it.Seek(keyOrPrefix); it.Valid(); it.Next())
{
yield return(it.Key(), it.Value());
}
}
else
{
for (it.SeekForPrev(keyOrPrefix); it.Valid(); it.Prev())
{
yield return(it.Key(), it.Value());
}
}
}
public void AddLogEntryByFollower(StateLogEntrySuggestion suggestion)
{
//remove all log bigger than this,(clear no committed logs)
var key = GetKey(suggestion.StateLogEntry.Term, suggestion.StateLogEntry.Index);
var iter = db.NewIterator();
iter.Seek(key);
while (iter.Valid())
{
iter.Next();
if (iter.Valid())
{
db.Remove(iter.Key());
}
else
{
break;
}
}
//add this one
AddLogEntry(suggestion);
//update commit status
if (suggestion.IsCommitted)
{
if (this.LastCommittedIndexTerm > suggestion.StateLogEntry.Term ||
(
this.LastCommittedIndexTerm == suggestion.StateLogEntry.Term &&
this.LastCommittedIndex > suggestion.StateLogEntry.Index
))
{
//Should be not possible
}
else
{
this.LastCommittedIndex = suggestion.StateLogEntry.Index;
this.LastCommittedIndexTerm = suggestion.StateLogEntry.Term;
}
}
}
public IActionResult Tally(List <IFormFile> files)
{
if (!HttpContext.Session.Keys.Contains(IsAuthenticated))
{
return(RedirectToAction("Signin"));
}
if (files.Count != 5)
{
return(RedirectToAction("Tally", new { msg = "Missing keys" }));
}
var kk = new List <string>();
foreach (var f in files)
{
kk.Add((new StreamReader(f.OpenReadStream())).ReadToEnd());
}
if (!JoinKeys(kk))
{
return(RedirectToAction("Tally", new { msg = "Fail to obtain master key" }));
}
ElectionGuard.ElectionDescription eldesc;
lock (_conf)
{
if (_conf.Get(ESElectionConfigurationKey) == null)
{
throw new Exception("ESConfiguration missing");
}
eldesc = JsonSerializer.Deserialize <ElectionGuard.ElectionDescription>(_conf.Get(ESElectionConfigurationKey));
}
var protector = dataProtector.CreateProtector("SecureBallot");
var ballots = new List <BallotContent>();
lock (secureBallot)
{
if (secureBallot.Get(VotingForTallyClosedKey) == null)
{
secureBallot.Put(VotingForTallyClosedKey, DateTime.Now.ToString());
}
using (var it = secureBallot.NewIterator())
{
it.SeekToFirst();
while (it.Valid())
{
// This is the only exception and it is the seal of the Ballot
if (it.StringKey() != HomeController.VotingForTallyClosedKey)
{
ballots.Add(BallotContent.FromJson(it.StringValue()));
}
it.Next();
}
}
}
var clearBallots = ballots.ConvertAll(b => (b.ElectionId, protector.Unprotect(b.SecureVote)));
var result = new Dictionary <string, Dictionary <string, int> >();
foreach (var ballot in clearBallots.GroupBy(b => b.ElectionId))
{
var count = ballot.GroupBy(b => b.Item2);
foreach (var c in count)
{
if (!result.ContainsKey(ballot.Key))
{
result.Add(ballot.Key, new Dictionary <string, int>());
}
result[ballot.Key].Add(c.Key, c.Count());
}
}
var elections = eldesc.contests.ToDictionary(g => g.object_id);
return(View((result, elections)));
}
/// <inheritdoc />
public GarbageCollectResult GarbageCollectByKeyValue(
Func <Iterator, bool> canCollect,
string columnFamilyName = null,
IEnumerable <string> additionalColumnFamilies = null,
CancellationToken cancellationToken = default,
byte[] startValue = null)
{
var gcStats = new GarbageCollectResult
{
BatchSize = GarbageCollectionBatchSize
};
var columnFamilyInfo = GetColumnFamilyInfo(columnFamilyName);
var columnFamilyHandleToUse = columnFamilyInfo.Handle;
// According to RocksDB documentation, an iterator always loads both the key
// and the value. To avoid this, when possible, we use the key-tracked column with just keys
// and empty values and use that for eviction to avoid the load cost of full content.
if (columnFamilyInfo.UseKeyTracking)
{
columnFamilyHandleToUse = columnFamilyInfo.KeyHandle;
}
var keysToRemove = new List <byte[]>();
var primaryColumn = new string[] { columnFamilyName };
var columnsToUse = additionalColumnFamilies == null ? primaryColumn : additionalColumnFamilies.Concat(primaryColumn);
using (Iterator iterator = m_store.NewIterator(columnFamilyHandleToUse, m_readOptions))
{
if (startValue != null)
{
iterator.Seek(startValue);
}
else
{
iterator.SeekToFirst();
}
bool reachedEnd = !iterator.Valid();
while (!reachedEnd && !cancellationToken.IsCancellationRequested)
{
gcStats.TotalCount++;
bool canCollectResult = canCollect(iterator);
if (canCollectResult)
{
var bytesKey = iterator.Key();
keysToRemove.Add(bytesKey);
}
iterator.Next();
reachedEnd = !iterator.Valid();
if (keysToRemove.Count == GarbageCollectionBatchSize ||
(reachedEnd && keysToRemove.Count > 0))
{
var startTime = TimestampUtilities.Timestamp;
// Remove the key across all specified columns
RemoveBatch(keysToRemove, columnFamilyNames: columnsToUse);
var duration = TimestampUtilities.Timestamp - startTime;
if (duration > gcStats.MaxBatchEvictionTime)
{
gcStats.MaxBatchEvictionTime = duration;
}
gcStats.LastKey = keysToRemove.Last();
gcStats.RemovedCount += keysToRemove.Count;
keysToRemove.Clear();
}
}
}
gcStats.Canceled = cancellationToken.IsCancellationRequested;
return(gcStats);
}
public IEnumerable <byte[]> GetAll()
{
using Iterator iterator = _rocksDb.NewIterator(_columnFamily);
return(_mainDb.GetAllCore(iterator));
}