public IStoreGraph CreateGraph(string id, GraphType type)
{
var key = MakeKey(id, type);
_db.Put(key, "1");
return(new RocksGraph(key, _db));
}
public void PutPageContent(string title, ContentPageMetadata meta, byte[] pageContent)
{
if (title == null || meta == null)
{
return;
}
var serializedMetadata = _dbInstance.Get(Encoding.UTF8.GetBytes($"{title}@Metadata"));
if (serializedMetadata != null)
{
using (var metadataStream = new MemoryStream(serializedMetadata))
{
var prevMetadata = (ContentPageMetadata)_formatter.Deserialize(metadataStream);
if (prevMetadata.ChangeSetId > meta.ChangeSetId)
{
// Update is not required
_logger.LogWarning($"Cache update request for {title} has rev {prevMetadata.ChangeSetId} > {meta.ChangeSetId}");
return;
}
}
}
using (var metadataStream = new MemoryStream())
{
_formatter.Serialize(metadataStream, meta);
_dbInstance.Put(Encoding.UTF8.GetBytes($"{title}@Metadata"), metadataStream.GetBuffer());
_dbInstance.Put(Encoding.UTF8.GetBytes($"{title}@Content"), pageContent);
}
}
/*
* Put(byte[] key, byte[] value);
* Put(string key, string value);
* Put(byte[] key, long keyLength, byte[] value, long valueLength);
*
* Get(byte[] key, byte[] buffer, long offset, long length);----------long
* Get(string key);----------string
* Get(byte[] key, long keyLength, byte[] buffer, long offset, long length);----------long
* Get(byte[] key, long keyLength);----------byte[]
* Get(byte[] key);----------byte[]
*
* Remove(byte[] key);
* Remove(string key);
* Remove(byte[] key, long keyLength);
*/
//Put
public void Put(string txid, int n, string blockIndex, string asset, string address, string value)
{
string key = txid + n;
string utxo = "{\"blockIndex\":\"" + blockIndex + "\",\"asset\":\"" + asset + "\",\"address\":\"" + address + "\",\"value\":\"" + value + "\"}";
db.Put(key, utxo);
}
public void Put(string key, string value)
{
var exists = _rocksDb.Get(key);
if (exists.xIsEmpty())
{
_rocksDb.Put(key, value);
}
}
public async Task IngestSamplesAsync(List <ContribSampleEntity> samples, CancellationToken cancellationToken)
{
if (_bypassLocalCache)
{
throw new NotSupportedException();
}
// Cutoff time span is 1 hour
// Other potion are assumed aggregated and directly ingested into the database
// The rest are not aggregated and will retain in the RocksDb for a while
var currentTime = DateTime.UtcNow;
var cutoffTime = new DateTime(currentTime.Year, currentTime.Month,
currentTime.Day, currentTime.Hour, 0, 0, DateTimeKind.Utc);
var xTableWritePotion = new List <ContribSampleEntity>();
var aggDictionary = new ConcurrentDictionary <string, long>();
foreach (var sample in samples)
{
if (sample.MetricTimeStampUtc < cutoffTime)
{
xTableWritePotion.Add(sample);
}
else
{
aggDictionary.AddOrUpdate(sample.PartitionKey, sample.Count,
(key, value) => value + sample.Count);
}
}
// Ingest into XTable
await IngestSamplesToXTableAsync(xTableWritePotion, cancellationToken);
// Cache the 1hr data
foreach (var k in aggDictionary)
{
long count;
var kb = Encoding.UTF8.GetBytes(k.Key);
var counterBytes = _preAggDatabase.Get(kb);
if (counterBytes != null)
{
count = (long)_formatter.Deserialize(new MemoryStream(counterBytes));
count += k.Value;
}
else
{
count = k.Value;
}
var s = new MemoryStream();
_formatter.Serialize(s, count);
_preAggDatabase.Put(kb, s.GetBuffer());
}
}
public void Put(byte[]?key, byte[] value)
{
if (disposed || db.Handle == IntPtr.Zero)
{
throw new ObjectDisposedException(nameof(RocksDbStore));
}
if (readOnly)
{
throw new InvalidOperationException("read only");
}
db.Put(key ?? Array.Empty <byte>(), value, columnFamily, writeOptions);
}
public async Task <string> GetHtml(string url, Param[] parameters = null)
{
parameters?.ForEach(p => url = url.Replace("{" + p.Key + "}", p.Value));
while (true)
{
try
{
Logger.LogDebug("Scrapping " + url + "...");
var result = _db.Get(url);
if (!string.IsNullOrWhiteSpace(result))
{
Metrics.Inc("pump_httpextractor_nbloadedfromcache", 1);
Logger.LogDebug(url + " => Loaded from cache :):):)");
return(result);
}
var handler = new HttpClientHandler
{
ClientCertificateOptions = ClientCertificateOption.Manual,
ServerCertificateCustomValidationCallback =
(httpRequestMessage, cert, cetChain, policyErrors) => true
};
using (var client = new HttpClient(handler))
{
Logger.LogDebug(url + " => Not found in cache");
var response = await client.GetAsync(url);
Logger.LogDebug(url + " => Http request done : " + response.StatusCode);
Metrics.Inc("pump_httpextractor_bytesloaded", response.Content.Headers.ContentLength ?? 0);
result = await response.Content.ReadAsStringAsync();
result = result.Replace("\n", "")
.Replace("\t", "")
.Replace("\\\"", "\"");
_db.Put(url, result);
Metrics.Inc("pump_httpextractor_nbloadedfromhttp", 1);
Logger.LogDebug(url + " => Loaded from http");
return(result);
}
}
catch (Exception e)
{
Metrics.Inc("pump_httpextractor_errors", 1);
Logger.LogError(e.GetFullMessage());
Thread.Sleep(60000);
}
}
}
/// <inheritdoc/>
public override void PutTransaction <T>(Transaction <T> tx)
{
if (_txCache.ContainsKey(tx.Id))
{
return;
}
byte[] key = TxKey(tx.Id);
if (!(_txIndexDb.Get(key) is null))
{
return;
}
long timestamp = tx.Timestamp.ToUnixTimeSeconds();
string txDbName = $"epoch{(int)timestamp / _txEpochUnitSeconds}";
_rwTxLock.EnterWriteLock();
try
{
if (!_txDbCache.TryGetValue(txDbName, out RocksDb txDb))
{
txDb = RocksDBUtils.OpenRocksDb(_options, TxDbPath(txDbName));
_txDbCache.AddOrUpdate(txDbName, txDb);
}
txDb.Put(key, tx.Serialize(true));
_txIndexDb.Put(key, RocksDBStoreBitConverter.GetBytes(txDbName));
_txCache.AddOrUpdate(tx.Id, tx);
}
finally
{
_rwTxLock.ExitWriteLock();
}
}
/// <inheritdoc cref="BaseStore.PutTxIdBlockHashIndex(TxId, BlockHash)"/>
public override void PutTxIdBlockHashIndex(TxId txId, BlockHash blockHash)
{
_txIdBlockHashIndexDb.Put(
TxIdBlockHashIndexKey(txId, blockHash),
blockHash.ToByteArray()
);
}
public void Put(string key, string value)
{
var bkey = Encoding.UTF8.GetBytes(key);
var bvalue = Encoding.UTF8.GetBytes(value);
rocksdb.Put(bkey, bvalue);
}
public byte[] this[byte[] key]
{
get
{
UpdateReadMetrics();
return(_db.Get(key));
}
set
{
UpdateWriteMetrics();
if (_currentBatch != null)
{
if (value == null)
{
_currentBatch.Delete(key);
}
else
{
_currentBatch.Put(key, value);
}
}
else
{
if (value == null)
{
_db.Remove(key);
}
else
{
_db.Put(key, value);
}
}
}
}
internal bool TrySetTerm(long term)
{
if (term > Term)
{
Term = term;
var termValue = ByteBuffer.Allocate();
termValue.WriteLong(term);
Rafts.Put(
RaftsTermKey, RaftsTermKey.Length,
termValue.Bytes, termValue.Size,
null, new WriteOptions().SetSync(true)
);
return(true);
}
return(false);
}
public byte[] this[byte[] key]
{
get
{
return(Db.Get(key));
}
set
{
UpdateWriteMetrics();
if (CurrentBatch != null)
{
if (value == null)
{
CurrentBatch.Delete(key);
}
else
{
CurrentBatch.Put(key, value);
}
}
else
{
if (value == null)
{
Db.Remove(key, null, WriteOptions);
}
else
{
Db.Put(key, value, null, WriteOptions);
}
}
}
}
public byte[] this[byte[] key]
{
get
{
UpdateReadMetrics();
return(_rocksDb.Get(key, _columnFamily));
}
set
{
UpdateWriteMetrics();
if (_mainDb.CurrentBatch != null)
{
if (value == null)
{
_mainDb.CurrentBatch.Delete(key, _columnFamily);
}
else
{
_mainDb.CurrentBatch.Put(key, value, _columnFamily);
}
}
else
{
if (value == null)
{
_rocksDb.Remove(key, _columnFamily, _mainDb.WriteOptions);
}
else
{
_rocksDb.Put(key, value, _columnFamily, _mainDb.WriteOptions);
}
}
}
}
public Store(string path)
{
var families = new ColumnFamilies();
try
{
foreach (var family in RocksDb.ListColumnFamilies(Options.Default, Path.GetFullPath(path)))
{
families.Add(new ColumnFamilies.Descriptor(family, new ColumnFamilyOptions()));
}
}
catch { }
db = RocksDb.Open(Options.Default, Path.GetFullPath(path), families);
ColumnFamilyHandle defaultFamily = db.GetDefaultColumnFamily();
byte[] value = db.Get(SYS_Version, defaultFamily, Options.ReadDefault);
if (value != null && Version.TryParse(Encoding.ASCII.GetString(value), out Version version) && version >= Version.Parse("3.0.0"))
{
return;
}
if (value != null)
{
// Clean all families only if the version are different
Parallel.For(0, byte.MaxValue + 1, (x) => db.DropColumnFamily(x.ToString()));
_families.Clear();
}
// Update version
db.Put(SYS_Version, Encoding.ASCII.GetBytes(Assembly.GetExecutingAssembly().GetName().Version.ToString()), defaultFamily, Options.WriteDefault);
}
private void PutBytes(string key, string value)
{
var eKey = Encoding.UTF8.GetBytes(key);
var eVal = Encoding.UTF8.GetBytes(value);
_db.Put(eKey, eVal);
}
private static void RocksDb2()
{
string directory = @"C:\Users\Peska\source\repos\Esent.Tests\RocksDB";
DbOptions options = new DbOptions().SetCreateIfMissing();
using (RocksDb rocksDb = RocksDbSharp.RocksDb.Open(options, directory))
{
TestObject o1 = new TestObject {
EQNum = "3L1234", Mnemonic = "20013L1234011"
};
TestObject o2 = new TestObject {
EQNum = "3L5678", Mnemonic = "20023L5678011"
};
TestObject o3 = new TestObject {
EQNum = "3L9012", Mnemonic = "20033L9012011"
};
TestObject o4 = new TestObject {
EQNum = "3L9012", Mnemonic = "20013L9012012"
};
rocksDb.Put(o1.Mnemonic, JsonConvert.SerializeObject(o1));
rocksDb.Put(o2.Mnemonic, JsonConvert.SerializeObject(o2));
rocksDb.Put(o3.Mnemonic, JsonConvert.SerializeObject(o3));
rocksDb.Put(o4.Mnemonic, JsonConvert.SerializeObject(o4));
SliceTransform sliceTransform = SliceTransform.CreateFixedPrefix(4);
BlockBasedTableOptions blockBasedTableOptions = new BlockBasedTableOptions().SetWholeKeyFiltering(false);
ColumnFamilyOptions columnFamilyOptions = new ColumnFamilyOptions()
.SetPrefixExtractor(sliceTransform)
.SetBlockBasedTableFactory(blockBasedTableOptions);
ColumnFamilies columnFamilies = new ColumnFamilies(columnFamilyOptions);
Iterator iterator = rocksDb.NewIterator();
iterator = iterator.Seek("2001");
while (iterator.Valid())
{
string key = iterator.StringKey();
string value = iterator.StringValue();
iterator.Next();
}
}
}
public byte[] this[byte[] key]
{
get
{
switch (_dbInstance)
{
case DbInstance.DiscoveryNodes:
Metrics.DiscoveryNodesDbReads++;
break;
case DbInstance.Peers:
Metrics.PeersDbReads++;
break;
}
if (_currentBatch != null)
{
return(_currentBatch.Get(key));
}
return(_db.Get(key));
}
set
{
switch (_dbInstance)
{
case DbInstance.DiscoveryNodes:
Metrics.DiscoveryNodesDbWrites++;
break;
case DbInstance.Peers:
Metrics.PeersDbWrites++;
break;
}
if (_currentBatch != null)
{
if (value == null)
{
_currentBatch.Delete(key);
}
else
{
_currentBatch.Put(key, value);
}
}
else
{
if (value == null)
{
_db.Remove(key);
}
else
{
_db.Put(key, value);
}
}
}
}
public void Put <T>(string key, T item)
{
using (var objectStream = new MemoryStream())
{
_formatter.Serialize(objectStream, item);
_dbInstance.Put(Encoding.UTF8.GetBytes(key), objectStream.GetBuffer());
}
}
/// <inheritdoc/>
public override void StageTransactionIds(IImmutableSet <TxId> txids)
{
foreach (TxId txId in txids)
{
byte[] key = StagedTxKey(txId);
_stagedTxDb.Put(key, EmptyBytes);
}
}
public static void Put(this RocksDb db, byte table, byte[] key, byte[] value)
{
Span <byte> dbkey = stackalloc byte[key.Length + 1];
dbkey[0] = table;
key.AsSpan().CopyTo(dbkey.Slice(1));
db.Put(dbkey.ToArray(), value);
}
public IActionResult LoadESElectionConfiguration()
{
if (!HttpContext.Session.Keys.Contains(IsAuthenticated))
{
return(RedirectToAction("Signin"));
}
var confAPI = new VotingSystemConfiguration();
lock (_conf)
{
if (_conf.Get(ESElectionConfigurationKey) != null)
{
return(new ForbidResult());
}
var v = _conf.Get(APIConfigurationKey);
if (v != null)
{
confAPI = VotingSystemConfiguration.FromJson(v);
}
}
var urlBuilder = new System.Text.StringBuilder();
urlBuilder.Append(confAPI.ElectionSystemAPI.TrimEnd('/')).Append("/RunningElections");
var req = WebRequest.Create(urlBuilder.ToString());
var resp = req.GetResponse();
var pdata = (new StreamReader(resp.GetResponseStream())).ReadToEnd();
var dp = dataProtector.CreateProtector("EligereMetadataExchange");
var data = dp.Unprotect(pdata);
var fn = Path.Combine(contentRootPath, $"wwwroot/temp/log.txt");
System.IO.File.WriteAllText(fn, data);
var eldesc = JsonSerializer.Deserialize <ElectionGuard.ElectionDescription>(data);
lock (_conf)
{
_conf.Put(ESElectionConfigurationKey, JsonSerializer.Serialize <ElectionGuard.ElectionDescription>(eldesc));
}
return(View("ShowESElectionConfiguration", eldesc));
}
private static void RocksDb1()
{
string directory = @"C:\Users\Peska\source\repos\Esent.Tests\RocksDB";
int maxSize = 1_000_000;
TestObject[] objects = new TestObject[maxSize];
for (int i = 0; i < maxSize; i++)
{
objects[i] = TestObject.Create();
if (i % 1000 == 0)
{
Console.WriteLine($"Created {i} records");
}
}
DbOptions options = new DbOptions().SetCreateIfMissing();
using (RocksDb rocksDb = RocksDbSharp.RocksDb.Open(options, directory))
{
for (int i = 0; i < maxSize; i++)
{
string json = JsonConvert.SerializeObject(objects[i]);
rocksDb.Put(objects[i].EQNum, json);
if (i % 1000 == 0)
{
Console.WriteLine($"Indexed {i} records");
}
}
Random random = new Random();
for (int i = 0; i < maxSize; i++)
{
int randomInt = random.Next(maxSize - 1);
TestObject testObject = objects[randomInt];
TestObject fromDict = JsonConvert.DeserializeObject <TestObject>(rocksDb.Get(testObject.EQNum));
if (testObject.EQNum != fromDict.EQNum)
{
throw new Exception("What the f**k");
}
if (i % 1000 == 0)
{
Console.WriteLine($"Retrieved {i} records");
}
}
Iterator iterator = rocksDb.NewIterator();
}
}
/// <summary>
/// Update or Insert Key's value from Rocksdb
/// </summary>
/// <param name="Key">Search Key</param>
/// <returns></returns>
bool PutDb(string Key, string Value)
{
try {
db.Put(Key, Value);
return(true);
} catch (Exception exc) {
Log.Error(exc, $"PutDb(key:{Key}, value:{Value})");
return(false);
}
}
/// <inheritdoc/>
public override void DeleteChainId(Guid chainId)
{
ColumnFamilyHandle cf = GetColumnFamily(_chainDb, chainId);
if (HasFork(chainId))
{
_chainDb.Put(DeletedKey, new byte[0], cf);
// We need only chain indexes, not tx nonces at this time because they already had
// been copied on .ForkTxNonces().
// FIXME: We should remove this code after adjusting .ForkTxNonces().
using var batch = new WriteBatch();
foreach (Iterator k in IterateDb(_chainDb, TxNonceKeyPrefix, chainId))
{
batch.Delete(k.Key(), cf);
}
_chainDb.Write(batch);
return;
}
_logger.Debug($"Deleting chainID: {chainId}.");
Guid?prevChain = GetPreviousChainInfo(cf)?.Item1;
string cfName = chainId.ToString();
try
{
_chainDb.DropColumnFamily(cfName);
if (prevChain is { } prevChainNotNull)
{
lock (_chainForkDeleteLock)
{
if (HasFork(prevChainNotNull))
{
ColumnFamilyHandle prevCf = GetColumnFamily(_chainDb, prevChainNotNull);
RemoveFork(prevCf, chainId);
if (IsDeletionMarked(prevCf))
{
DeleteChainId(prevChainNotNull);
}
}
}
}
}
catch (KeyNotFoundException)
{
// Do nothing according to the specification: DeleteChainId() should be idempotent.
_logger.Debug($"No such chain ID in _chainDb: {cfName}.", cfName);
}
catch (Exception e)
{
LogUnexpectedException(nameof(DeleteChainId), e);
}
}
public string[] ElectionGuardGenGuardians(int numGuardians, int quorum)
{
// ElectionGuard generation
ElectionGuard.ElectionDescription eldesc;
lock (_conf)
{
if (_conf.Get(ESElectionConfigurationKey) == null)
{
throw new Exception("ESConfiguration missing");
}
eldesc = JsonSerializer.Deserialize <ElectionGuard.ElectionDescription>(_conf.Get(ESElectionConfigurationKey));
}
var guardianIds = new string[numGuardians];
for (var i = 0; i < guardianIds.Length; i++)
{
guardianIds[i] = $"Guardian-{i}";
}
var guardians = new List <ElectionGuard.Guardian>();
for (var i = 0; i < numGuardians; i++)
{
guardians.Add(GuardianApi.Guardian(null, null, guardianIds[i], numGuardians, quorum, i));
}
var guardianPubKeys = guardians.ConvertAll(g => g.election_key_pair.public_key).ToArray();
var electionJointKey = MediatorApi.ElectionCombine(guardianPubKeys);
var electionContext = MediatorApi.ElectionContext(eldesc, electionJointKey.joint_key, numGuardians, quorum);
// Persists only the public part
lock (egSecureBallot)
{
egSecureBallot.Put("GuardianPubKeys", JsonSerializer.Serialize(guardianPubKeys));
egSecureBallot.Put("ElectionJointKey", JsonSerializer.Serialize(electionJointKey));
egSecureBallot.Put("ElectionContext", JsonSerializer.Serialize(electionContext));
egSecureBallot.Put("Nonce", "110191403412906482859082647039385408757148325819889522238592336039604240167009");
egSecureBallot.Put("SeedHash", "110191403412906482859082647039385908787142225838889522238592336039604240167009");
}
return(guardians.ConvertAll(g => JsonSerializer.Serialize(g)).ToArray());
}
/// <inheritdoc/>
public override void SetBlockPerceivedTime(
HashDigest <SHA256> blockHash,
DateTimeOffset perceivedTime
)
{
byte[] key = BlockKey(blockHash);
_blockPerceptionDb.Put(
key,
NetworkOrderBitsConverter.GetBytes(perceivedTime.ToUnixTimeMilliseconds())
);
}
private static void WriteHashCache(AbsolutePath file, Hash hash)
{
using var ms = new MemoryStream(20);
using var bw = new BinaryWriter(ms);
bw.Write(HashCacheVersion);
var lastModified = file.LastModifiedUtc.AsUnixTime();
bw.Write(lastModified);
bw.Write((ulong)hash);
_hashCache.Put(Encoding.UTF8.GetBytes(file.Normalize()), ms.ToArray());
}
public void create_rocksdb_test()
{
Parallel.ForEach(Enumerable.Range(1, 10), i => {
// Using strings below, but can also use byte arrays for both keys and values
// much care has been taken to minimize buffer copying
_rocksDb.Put($"key{i}", "value");
string value = _rocksDb.Get($"key{i}");
Assert.AreEqual("value", value);
_rocksDb.Remove($"key{i}");
});
_rocksDb.Dispose();
}
/// <inheritdoc/>
public override void PutBlock <T>(Block <T> block)
{
if (_blockCache.ContainsKey(block.Hash))
{
return;
}
byte[] key = BlockKey(block.Hash);
if (!(_blockIndexDb.Get(key) is null))
{
return;
}
long timestamp = block.Timestamp.ToUnixTimeSeconds();
foreach (Transaction <T> tx in block.Transactions)
{
PutTransaction(tx);
}
_rwBlockLock.EnterWriteLock();
try
{
string blockDbName = $"epoch{timestamp / _blockEpochUnitSeconds}";
RocksDb blockDb;
lock (_blockDbCache)
{
if (!_blockDbCache.TryGetValue(blockDbName, out blockDb))
{
blockDb = RocksDBUtils.OpenRocksDb(_options, BlockDbPath(blockDbName));
_blockDbCache.AddOrUpdate(blockDbName, blockDb);
}
}
BlockDigest digest = BlockDigest.FromBlock(block);
byte[] value = digest.Serialize();
blockDb.Put(key, value);
_blockIndexDb.Put(key, RocksDBStoreBitConverter.GetBytes(blockDbName));
_blockCache.AddOrUpdate(block.Hash, digest);
}
catch (Exception e)
{
LogUnexpectedException(nameof(PutBlock), e);
}
finally
{
_rwBlockLock.ExitWriteLock();
}
}