/// <inheritdoc/>
public override BlockDigest?GetBlockDigest(HashDigest <SHA256> blockHash)
{
if (_blockCache.TryGetValue(blockHash, out BlockDigest cachedDigest))
{
return(cachedDigest);
}
byte[] key = BlockKey(blockHash);
if (!(_blockIndexDb.Get(key) is byte[] blockDbNameBytes))
{
return(null);
}
_rwBlockLock.EnterReadLock();
try
{
string blockDbName = RocksDBStoreBitConverter.GetString(blockDbNameBytes);
if (!_blockDbCache.TryGetValue(blockDbName, out RocksDb blockDb))
{
blockDb = RocksDBUtils.OpenRocksDb(_options, BlockDbPath(blockDbName));
_blockDbCache.AddOrUpdate(blockDbName, blockDb);
}
byte[] blockBytes = blockDb.Get(key);
BlockDigest blockDigest = BlockDigest.Deserialize(blockBytes);
_blockCache.AddOrUpdate(blockHash, blockDigest);
return(blockDigest);
}
finally
{
_rwBlockLock.ExitReadLock();
}
}
public ulong GetCounterValue(Counter counter, Span <byte> index)
{
using var _ = RocksDbEncoder.EncodeCounter(counter, index, out var key);
var result = Database.Get(key);
return(result != null?BinaryPrimitives.ReadUInt64LittleEndian(result) : 0);
}
public byte[] this[byte[] key]
{
get
{
UpdateReadMetrics();
return(_db.Get(key));
}
set
{
UpdateWriteMetrics();
if (_currentBatch != null)
{
if (value == null)
{
_currentBatch.Delete(key);
}
else
{
_currentBatch.Put(key, value);
}
}
else
{
if (value == null)
{
_db.Remove(key);
}
else
{
_db.Put(key, value);
}
}
}
}
public byte[] this[byte[] key]
{
get
{
UpdateReadMetrics();
return(_rocksDb.Get(key, _columnFamily));
}
set
{
UpdateWriteMetrics();
if (_mainDb.CurrentBatch != null)
{
if (value == null)
{
_mainDb.CurrentBatch.Delete(key, _columnFamily);
}
else
{
_mainDb.CurrentBatch.Put(key, value, _columnFamily);
}
}
else
{
if (value == null)
{
_rocksDb.Remove(key, _columnFamily, _mainDb.WriteOptions);
}
else
{
_rocksDb.Put(key, value, _columnFamily, _mainDb.WriteOptions);
}
}
}
}
/// <inheritdoc />
public override Guid?GetCanonicalChainId()
{
byte[] bytes = _chainDb.Get(CanonicalChainIdIdKey);
return(bytes is null
? (Guid?)null
: new Guid(bytes));
}
public void Put(string key, string value)
{
var exists = _db.Get(key);
if (exists.xIsEmpty())
{
PutBytes(key, value);
}
}
public byte[] GetPageContent(string title)
{
if (title == null)
{
return(null);
}
return(_dbInstance.Get(Encoding.UTF8.GetBytes($"{title}@Content")));
}
public void Put(string key, string value)
{
var exists = _rocksDb.Get(key);
if (exists.xIsEmpty())
{
_rocksDb.Put(key, value);
}
}
public async Task IngestSamplesAsync(List <ContribSampleEntity> samples, CancellationToken cancellationToken)
{
if (_bypassLocalCache)
{
throw new NotSupportedException();
}
// Cutoff time span is 1 hour
// Other potion are assumed aggregated and directly ingested into the database
// The rest are not aggregated and will retain in the RocksDb for a while
var currentTime = DateTime.UtcNow;
var cutoffTime = new DateTime(currentTime.Year, currentTime.Month,
currentTime.Day, currentTime.Hour, 0, 0, DateTimeKind.Utc);
var xTableWritePotion = new List <ContribSampleEntity>();
var aggDictionary = new ConcurrentDictionary <string, long>();
foreach (var sample in samples)
{
if (sample.MetricTimeStampUtc < cutoffTime)
{
xTableWritePotion.Add(sample);
}
else
{
aggDictionary.AddOrUpdate(sample.PartitionKey, sample.Count,
(key, value) => value + sample.Count);
}
}
// Ingest into XTable
await IngestSamplesToXTableAsync(xTableWritePotion, cancellationToken);
// Cache the 1hr data
foreach (var k in aggDictionary)
{
long count;
var kb = Encoding.UTF8.GetBytes(k.Key);
var counterBytes = _preAggDatabase.Get(kb);
if (counterBytes != null)
{
count = (long)_formatter.Deserialize(new MemoryStream(counterBytes));
count += k.Value;
}
else
{
count = k.Value;
}
var s = new MemoryStream();
_formatter.Serialize(s, count);
_preAggDatabase.Put(kb, s.GetBuffer());
}
}
public async Task <string> GetHtml(string url, Param[] parameters = null)
{
parameters?.ForEach(p => url = url.Replace("{" + p.Key + "}", p.Value));
while (true)
{
try
{
Logger.LogDebug("Scrapping " + url + "...");
var result = _db.Get(url);
if (!string.IsNullOrWhiteSpace(result))
{
Metrics.Inc("pump_httpextractor_nbloadedfromcache", 1);
Logger.LogDebug(url + " => Loaded from cache :):):)");
return(result);
}
var handler = new HttpClientHandler
{
ClientCertificateOptions = ClientCertificateOption.Manual,
ServerCertificateCustomValidationCallback =
(httpRequestMessage, cert, cetChain, policyErrors) => true
};
using (var client = new HttpClient(handler))
{
Logger.LogDebug(url + " => Not found in cache");
var response = await client.GetAsync(url);
Logger.LogDebug(url + " => Http request done : " + response.StatusCode);
Metrics.Inc("pump_httpextractor_bytesloaded", response.Content.Headers.ContentLength ?? 0);
result = await response.Content.ReadAsStringAsync();
result = result.Replace("\n", "")
.Replace("\t", "")
.Replace("\\\"", "\"");
_db.Put(url, result);
Metrics.Inc("pump_httpextractor_nbloadedfromhttp", 1);
Logger.LogDebug(url + " => Loaded from http");
return(result);
}
}
catch (Exception e)
{
Metrics.Inc("pump_httpextractor_errors", 1);
Logger.LogError(e.GetFullMessage());
Thread.Sleep(60000);
}
}
}
private long ReadHighestSequenceNr(int persistenceId)
{
var ro = RocksDbSnapshot();
try
{
var num = database.Get(KeyToBytes(CounterKey(persistenceId)), cf: null, readOptions: ro);
return(num == null ? 0L : CounterFromBytes(num));
}
finally
{
//ro.Snapshot().Dispose();
}
}
public IActionResult ElectionCard(string tickets)
{
var encTickets = Convert.FromBase64String(tickets);
var protector = dataprotection.CreateProtector("EVSKeyExchange");
var plainSerTickets = Encoding.UTF8.GetString(protector.Unprotect(encTickets));
var ticketList = JsonSerializer.Deserialize <List <VoteTicket> >(plainSerTickets);
string d;
lock (configuration)
{
d = configuration.Get(HomeController.ESElectionConfigurationKey);
}
var electionDescription = JsonSerializer.Deserialize <ElectionGuard.ElectionDescription>(d);
var electionmap = electionDescription.contests.ToDictionary(v => v.object_id);
var elections2Send = new List <string>();
var usedTickets = new List <VoteTicket>();
var availableTickets = new List <VoteTicket>();
// Check used tickets
foreach (var ticket in ticketList)
{
if (!electionmap.ContainsKey(ticket.ElectionId))
{
throw new Exception("Invalid state in ElectionCard generation (ticket referring to non-existing election)");
}
elections2Send.Add(ticket.ElectionId);
if (ticketsDb.Get(ticket.HashId) != null)
{
usedTickets.Add(ticket);
}
else
{
availableTickets.Add(ticket);
}
}
electionDescription.contests = electionDescription.contests.Where(c => elections2Send.Contains(c.object_id)).ToArray();
var cardData = new VoteInformation()
{
ElectionDescription = electionDescription,
AvailableTickets = availableTickets,
UsedTickets = usedTickets
};
return(View((tickets, cardData)));
}
public string Get(string key)
{
var bkey = Encoding.UTF8.GetBytes(key);
var bvalue = rocksdb.Get(bkey);
return(Encoding.UTF8.GetString(bvalue));
}
public static bool TryGetHashCache(AbsolutePath file, out Hash hash)
{
var normPath = Encoding.UTF8.GetBytes(file.Normalize());
var value = _hashCache.Get(normPath);
hash = default;
if (value == null)
{
return(false);
}
if (value.Length != 20)
{
return(false);
}
using var ms = new MemoryStream(value);
using var br = new BinaryReader(ms);
var version = br.ReadUInt32();
if (version != HashCacheVersion)
{
return(false);
}
var lastModified = br.ReadUInt64();
if (lastModified != file.LastModifiedUtc.AsUnixTime())
{
return(false);
}
hash = new Hash(br.ReadUInt64());
return(true);
}
public byte[] this[byte[] key]
{
get
{
return(Db.Get(key));
}
set
{
UpdateWriteMetrics();
if (CurrentBatch != null)
{
if (value == null)
{
CurrentBatch.Delete(key);
}
else
{
CurrentBatch.Put(key, value);
}
}
else
{
if (value == null)
{
Db.Remove(key, null, WriteOptions);
}
else
{
Db.Put(key, value, null, WriteOptions);
}
}
}
}
public HomeController(ILogger <HomeController> logger, IWebHostEnvironment env, PersistentStores stores, IDataProtectionProvider provider)
{
_logger = logger;
contentRootPath = env.ContentRootPath;
stores.SetContentRootPath(env.ContentRootPath);
_conf = stores.Configuration;
secureBallot = stores.SecureBallot;
egSecureBallot = stores.EGSecureBallot;
dataProtector = provider;
var confAPI = new VotingSystemConfiguration();
lock (_conf)
{
var v = _conf.Get(APIConfigurationKey);
if (v != null)
{
confAPI = VotingSystemConfiguration.FromJson(v);
}
}
if (confAPI.GuardianAPI != null && confAPI.MediatorAPI != null)
{
GuardianApi = new ElectionGuard.GuardianClient(confAPI.GuardianAPI);
MediatorApi = new ElectionGuard.MediatorClient(confAPI.MediatorAPI);
}
}
public Triple SPI(string s, string p, int index)
{
var sh = KeySegments.GetNameSKeySubjectPredicateIndex(Name, s, p, index);
var t = _db.Get(sh);
return(t?.ToTriple());
}
public Store(string path)
{
var families = new ColumnFamilies();
try
{
foreach (var family in RocksDb.ListColumnFamilies(Options.Default, Path.GetFullPath(path)))
{
families.Add(new ColumnFamilies.Descriptor(family, new ColumnFamilyOptions()));
}
}
catch { }
db = RocksDb.Open(Options.Default, Path.GetFullPath(path), families);
ColumnFamilyHandle defaultFamily = db.GetDefaultColumnFamily();
byte[] value = db.Get(SYS_Version, defaultFamily, Options.ReadDefault);
if (value != null && Version.TryParse(Encoding.ASCII.GetString(value), out Version version) && version >= Version.Parse("3.0.0"))
{
return;
}
if (value != null)
{
// Clean all families only if the version are different
Parallel.For(0, byte.MaxValue + 1, (x) => db.DropColumnFamily(x.ToString()));
_families.Clear();
}
// Update version
db.Put(SYS_Version, Encoding.ASCII.GetBytes(Assembly.GetExecutingAssembly().GetName().Version.ToString()), defaultFamily, Options.WriteDefault);
}
public byte[] this[byte[] key]
{
get
{
switch (_dbInstance)
{
case DbInstance.DiscoveryNodes:
Metrics.DiscoveryNodesDbReads++;
break;
case DbInstance.Peers:
Metrics.PeersDbReads++;
break;
}
if (_currentBatch != null)
{
return(_currentBatch.Get(key));
}
return(_db.Get(key));
}
set
{
switch (_dbInstance)
{
case DbInstance.DiscoveryNodes:
Metrics.DiscoveryNodesDbWrites++;
break;
case DbInstance.Peers:
Metrics.PeersDbWrites++;
break;
}
if (_currentBatch != null)
{
if (value == null)
{
_currentBatch.Delete(key);
}
else
{
_currentBatch.Put(key, value);
}
}
else
{
if (value == null)
{
_db.Remove(key);
}
else
{
_db.Put(key, value);
}
}
}
}
public byte[] TryGet(byte[]?key)
{
if (snapshot.Handle == IntPtr.Zero)
{
throw new ObjectDisposedException(nameof(Snapshot));
}
return(db.Get(key ?? Array.Empty <byte>(), columnFamily, readOptions));
}
public byte[]? TryGet(byte[]?key)
{
if (disposed || db.Handle == IntPtr.Zero)
{
throw new ObjectDisposedException(nameof(RocksDbStore));
}
return(db.Get(key ?? Array.Empty <byte>(), columnFamily, readOptions));
}
/// <inheritdoc/>
public override Transaction <T> GetTransaction <T>(TxId txid)
{
if (_txCache.TryGetValue(txid, out object cachedTx))
{
return((Transaction <T>)cachedTx);
}
byte[] key = TxKey(txid);
if (!(_txIndexDb.Get(key) is byte[] txDbNameBytes))
{
return(null);
}
string txDbName = RocksDBStoreBitConverter.GetString(txDbNameBytes);
_rwTxLock.EnterReadLock();
try
{
RocksDb txDb;
lock (_txDbCache)
{
if (!_txDbCache.TryGetValue(txDbName, out txDb))
{
txDb = RocksDBUtils.OpenRocksDb(_options, TxDbPath(txDbName));
_txDbCache.AddOrUpdate(txDbName, txDb);
}
}
byte[] txBytes = txDb.Get(key);
Transaction <T> tx = Transaction <T> .Deserialize(txBytes, false);
_txCache.AddOrUpdate(txid, tx);
return(tx);
}
catch (Exception e)
{
LogUnexpectedException(nameof(GetTransaction), e);
return(null);
}
finally
{
_rwTxLock.ExitReadLock();
}
}
public static byte[] Get(this RocksDb db, byte table, byte[] key)
{
Span <byte> dbkey = stackalloc byte[key.Length + 1];
dbkey[0] = table;
key.AsSpan().CopyTo(dbkey.Slice(1));
return(db.Get(dbkey.ToArray()));
}
public static TValue?TryGet <TValue>(this RocksDb db, byte[] key, ColumnFamilyHandle?columnFamily = null, ReadOptions?readOptions = null)
where TValue : class, ISerializable, new()
{
var value = db.Get(key, columnFamily, readOptions);
return(value?.Length > 0
? value.AsSerializable <TValue>()
: null);
}
/// <inheritdoc cref="BaseStore.GetTxExecution(BlockHash, TxId)"/>
public override TxExecution GetTxExecution(BlockHash blockHash, TxId txid)
{
byte[] key = TxExecutionKey(blockHash, txid);
if (_txExecutionDb.Get(key) is { } bytes)
{
return(DeserializeTxExecution(blockHash, txid, Codec.Decode(bytes), _logger));
}
return(null);
}
/// <inheritdoc/>
public override Transaction <T> GetTransaction <T>(TxId txid)
{
if (_txCache.TryGetValue(txid, out object cachedTx))
{
return((Transaction <T>)cachedTx);
}
byte[] key = TxKey(txid);
byte[] bytes = _txDb.Get(key);
if (bytes is null)
{
return(null);
}
Transaction <T> tx = Transaction <T> .Deserialize(bytes, false);
_txCache.AddOrUpdate(txid, tx);
return(tx);
}
/// <summary>
/// Get Key's value from Rocksdb
/// </summary>
/// <param name="Key">Search Key</param>
/// <returns></returns>
string PostDb(string Key)
{
try {
string dbData = db.Get(Key);
return(dbData);
} catch (Exception exc) {
Log.Error(exc, $"PutDb(key:{Key})");
return("Error saved to logs");
}
}
/// <inheritdoc/>
public override BlockDigest?GetBlockDigest(HashDigest <SHA256> blockHash)
{
if (_blockCache.TryGetValue(blockHash, out BlockDigest cachedDigest))
{
return(cachedDigest);
}
byte[] key = BlockKey(blockHash);
byte[] bytes = _blockDb.Get(key);
if (bytes is null)
{
return(null);
}
BlockDigest blockDigest = BlockDigest.Deserialize(bytes);
_blockCache.AddOrUpdate(blockHash, blockDigest);
return(blockDigest);
}
/// <inheritdoc/>
public override DateTimeOffset?GetBlockPerceivedTime(HashDigest <SHA256> blockHash)
{
byte[] key = BlockKey(blockHash);
if (_blockPerceptionDb.Get(key) is { } bytes)
{
long unixTimeMs = NetworkOrderBitsConverter.ToInt64(bytes);
return(DateTimeOffset.FromUnixTimeMilliseconds(unixTimeMs));
}
return(null);
}
/// <summary>
/// Return the state value
/// </summary>
/// <param name="key">Key to identify the state</param>
/// <returns></returns>
public override object GetValue(string key)
{
object State = null;
try
{
State = Deserialize(Database.Get(Encoding.UTF8.GetBytes(key)));
}
catch (Exception) { }
return(State);
}