public IStoreGraph CreateGraph(string id, GraphType type) { var key = MakeKey(id, type); _db.Put(key, "1"); return(new RocksGraph(key, _db)); }
public void PutPageContent(string title, ContentPageMetadata meta, byte[] pageContent) { if (title == null || meta == null) { return; } var serializedMetadata = _dbInstance.Get(Encoding.UTF8.GetBytes($"{title}@Metadata")); if (serializedMetadata != null) { using (var metadataStream = new MemoryStream(serializedMetadata)) { var prevMetadata = (ContentPageMetadata)_formatter.Deserialize(metadataStream); if (prevMetadata.ChangeSetId > meta.ChangeSetId) { // Update is not required _logger.LogWarning($"Cache update request for {title} has rev {prevMetadata.ChangeSetId} > {meta.ChangeSetId}"); return; } } } using (var metadataStream = new MemoryStream()) { _formatter.Serialize(metadataStream, meta); _dbInstance.Put(Encoding.UTF8.GetBytes($"{title}@Metadata"), metadataStream.GetBuffer()); _dbInstance.Put(Encoding.UTF8.GetBytes($"{title}@Content"), pageContent); } }
/* * Put(byte[] key, byte[] value); * Put(string key, string value); * Put(byte[] key, long keyLength, byte[] value, long valueLength); * * Get(byte[] key, byte[] buffer, long offset, long length);----------long * Get(string key);----------string * Get(byte[] key, long keyLength, byte[] buffer, long offset, long length);----------long * Get(byte[] key, long keyLength);----------byte[] * Get(byte[] key);----------byte[] * * Remove(byte[] key); * Remove(string key); * Remove(byte[] key, long keyLength); */ //Put public void Put(string txid, int n, string blockIndex, string asset, string address, string value) { string key = txid + n; string utxo = "{\"blockIndex\":\"" + blockIndex + "\",\"asset\":\"" + asset + "\",\"address\":\"" + address + "\",\"value\":\"" + value + "\"}"; db.Put(key, utxo); }
public void Put(string key, string value) { var exists = _rocksDb.Get(key); if (exists.xIsEmpty()) { _rocksDb.Put(key, value); } }
public async Task IngestSamplesAsync(List <ContribSampleEntity> samples, CancellationToken cancellationToken) { if (_bypassLocalCache) { throw new NotSupportedException(); } // Cutoff time span is 1 hour // Other potion are assumed aggregated and directly ingested into the database // The rest are not aggregated and will retain in the RocksDb for a while var currentTime = DateTime.UtcNow; var cutoffTime = new DateTime(currentTime.Year, currentTime.Month, currentTime.Day, currentTime.Hour, 0, 0, DateTimeKind.Utc); var xTableWritePotion = new List <ContribSampleEntity>(); var aggDictionary = new ConcurrentDictionary <string, long>(); foreach (var sample in samples) { if (sample.MetricTimeStampUtc < cutoffTime) { xTableWritePotion.Add(sample); } else { aggDictionary.AddOrUpdate(sample.PartitionKey, sample.Count, (key, value) => value + sample.Count); } } // Ingest into XTable await IngestSamplesToXTableAsync(xTableWritePotion, cancellationToken); // Cache the 1hr data foreach (var k in aggDictionary) { long count; var kb = Encoding.UTF8.GetBytes(k.Key); var counterBytes = _preAggDatabase.Get(kb); if (counterBytes != null) { count = (long)_formatter.Deserialize(new MemoryStream(counterBytes)); count += k.Value; } else { count = k.Value; } var s = new MemoryStream(); _formatter.Serialize(s, count); _preAggDatabase.Put(kb, s.GetBuffer()); } }
public void Put(byte[]?key, byte[] value) { if (disposed || db.Handle == IntPtr.Zero) { throw new ObjectDisposedException(nameof(RocksDbStore)); } if (readOnly) { throw new InvalidOperationException("read only"); } db.Put(key ?? Array.Empty <byte>(), value, columnFamily, writeOptions); }
public async Task <string> GetHtml(string url, Param[] parameters = null) { parameters?.ForEach(p => url = url.Replace("{" + p.Key + "}", p.Value)); while (true) { try { Logger.LogDebug("Scrapping " + url + "..."); var result = _db.Get(url); if (!string.IsNullOrWhiteSpace(result)) { Metrics.Inc("pump_httpextractor_nbloadedfromcache", 1); Logger.LogDebug(url + " => Loaded from cache :):):)"); return(result); } var handler = new HttpClientHandler { ClientCertificateOptions = ClientCertificateOption.Manual, ServerCertificateCustomValidationCallback = (httpRequestMessage, cert, cetChain, policyErrors) => true }; using (var client = new HttpClient(handler)) { Logger.LogDebug(url + " => Not found in cache"); var response = await client.GetAsync(url); Logger.LogDebug(url + " => Http request done : " + response.StatusCode); Metrics.Inc("pump_httpextractor_bytesloaded", response.Content.Headers.ContentLength ?? 0); result = await response.Content.ReadAsStringAsync(); result = result.Replace("\n", "") .Replace("\t", "") .Replace("\\\"", "\""); _db.Put(url, result); Metrics.Inc("pump_httpextractor_nbloadedfromhttp", 1); Logger.LogDebug(url + " => Loaded from http"); return(result); } } catch (Exception e) { Metrics.Inc("pump_httpextractor_errors", 1); Logger.LogError(e.GetFullMessage()); Thread.Sleep(60000); } } }
/// <inheritdoc/> public override void PutTransaction <T>(Transaction <T> tx) { if (_txCache.ContainsKey(tx.Id)) { return; } byte[] key = TxKey(tx.Id); if (!(_txIndexDb.Get(key) is null)) { return; } long timestamp = tx.Timestamp.ToUnixTimeSeconds(); string txDbName = $"epoch{(int)timestamp / _txEpochUnitSeconds}"; _rwTxLock.EnterWriteLock(); try { if (!_txDbCache.TryGetValue(txDbName, out RocksDb txDb)) { txDb = RocksDBUtils.OpenRocksDb(_options, TxDbPath(txDbName)); _txDbCache.AddOrUpdate(txDbName, txDb); } txDb.Put(key, tx.Serialize(true)); _txIndexDb.Put(key, RocksDBStoreBitConverter.GetBytes(txDbName)); _txCache.AddOrUpdate(tx.Id, tx); } finally { _rwTxLock.ExitWriteLock(); } }
/// <inheritdoc cref="BaseStore.PutTxIdBlockHashIndex(TxId, BlockHash)"/> public override void PutTxIdBlockHashIndex(TxId txId, BlockHash blockHash) { _txIdBlockHashIndexDb.Put( TxIdBlockHashIndexKey(txId, blockHash), blockHash.ToByteArray() ); }
public void Put(string key, string value) { var bkey = Encoding.UTF8.GetBytes(key); var bvalue = Encoding.UTF8.GetBytes(value); rocksdb.Put(bkey, bvalue); }
public byte[] this[byte[] key] { get { UpdateReadMetrics(); return(_db.Get(key)); } set { UpdateWriteMetrics(); if (_currentBatch != null) { if (value == null) { _currentBatch.Delete(key); } else { _currentBatch.Put(key, value); } } else { if (value == null) { _db.Remove(key); } else { _db.Put(key, value); } } } }
internal bool TrySetTerm(long term) { if (term > Term) { Term = term; var termValue = ByteBuffer.Allocate(); termValue.WriteLong(term); Rafts.Put( RaftsTermKey, RaftsTermKey.Length, termValue.Bytes, termValue.Size, null, new WriteOptions().SetSync(true) ); return(true); } return(false); }
public byte[] this[byte[] key] { get { return(Db.Get(key)); } set { UpdateWriteMetrics(); if (CurrentBatch != null) { if (value == null) { CurrentBatch.Delete(key); } else { CurrentBatch.Put(key, value); } } else { if (value == null) { Db.Remove(key, null, WriteOptions); } else { Db.Put(key, value, null, WriteOptions); } } } }
public byte[] this[byte[] key] { get { UpdateReadMetrics(); return(_rocksDb.Get(key, _columnFamily)); } set { UpdateWriteMetrics(); if (_mainDb.CurrentBatch != null) { if (value == null) { _mainDb.CurrentBatch.Delete(key, _columnFamily); } else { _mainDb.CurrentBatch.Put(key, value, _columnFamily); } } else { if (value == null) { _rocksDb.Remove(key, _columnFamily, _mainDb.WriteOptions); } else { _rocksDb.Put(key, value, _columnFamily, _mainDb.WriteOptions); } } } }
public Store(string path) { var families = new ColumnFamilies(); try { foreach (var family in RocksDb.ListColumnFamilies(Options.Default, Path.GetFullPath(path))) { families.Add(new ColumnFamilies.Descriptor(family, new ColumnFamilyOptions())); } } catch { } db = RocksDb.Open(Options.Default, Path.GetFullPath(path), families); ColumnFamilyHandle defaultFamily = db.GetDefaultColumnFamily(); byte[] value = db.Get(SYS_Version, defaultFamily, Options.ReadDefault); if (value != null && Version.TryParse(Encoding.ASCII.GetString(value), out Version version) && version >= Version.Parse("3.0.0")) { return; } if (value != null) { // Clean all families only if the version are different Parallel.For(0, byte.MaxValue + 1, (x) => db.DropColumnFamily(x.ToString())); _families.Clear(); } // Update version db.Put(SYS_Version, Encoding.ASCII.GetBytes(Assembly.GetExecutingAssembly().GetName().Version.ToString()), defaultFamily, Options.WriteDefault); }
private void PutBytes(string key, string value) { var eKey = Encoding.UTF8.GetBytes(key); var eVal = Encoding.UTF8.GetBytes(value); _db.Put(eKey, eVal); }
private static void RocksDb2() { string directory = @"C:\Users\Peska\source\repos\Esent.Tests\RocksDB"; DbOptions options = new DbOptions().SetCreateIfMissing(); using (RocksDb rocksDb = RocksDbSharp.RocksDb.Open(options, directory)) { TestObject o1 = new TestObject { EQNum = "3L1234", Mnemonic = "20013L1234011" }; TestObject o2 = new TestObject { EQNum = "3L5678", Mnemonic = "20023L5678011" }; TestObject o3 = new TestObject { EQNum = "3L9012", Mnemonic = "20033L9012011" }; TestObject o4 = new TestObject { EQNum = "3L9012", Mnemonic = "20013L9012012" }; rocksDb.Put(o1.Mnemonic, JsonConvert.SerializeObject(o1)); rocksDb.Put(o2.Mnemonic, JsonConvert.SerializeObject(o2)); rocksDb.Put(o3.Mnemonic, JsonConvert.SerializeObject(o3)); rocksDb.Put(o4.Mnemonic, JsonConvert.SerializeObject(o4)); SliceTransform sliceTransform = SliceTransform.CreateFixedPrefix(4); BlockBasedTableOptions blockBasedTableOptions = new BlockBasedTableOptions().SetWholeKeyFiltering(false); ColumnFamilyOptions columnFamilyOptions = new ColumnFamilyOptions() .SetPrefixExtractor(sliceTransform) .SetBlockBasedTableFactory(blockBasedTableOptions); ColumnFamilies columnFamilies = new ColumnFamilies(columnFamilyOptions); Iterator iterator = rocksDb.NewIterator(); iterator = iterator.Seek("2001"); while (iterator.Valid()) { string key = iterator.StringKey(); string value = iterator.StringValue(); iterator.Next(); } } }
public byte[] this[byte[] key] { get { switch (_dbInstance) { case DbInstance.DiscoveryNodes: Metrics.DiscoveryNodesDbReads++; break; case DbInstance.Peers: Metrics.PeersDbReads++; break; } if (_currentBatch != null) { return(_currentBatch.Get(key)); } return(_db.Get(key)); } set { switch (_dbInstance) { case DbInstance.DiscoveryNodes: Metrics.DiscoveryNodesDbWrites++; break; case DbInstance.Peers: Metrics.PeersDbWrites++; break; } if (_currentBatch != null) { if (value == null) { _currentBatch.Delete(key); } else { _currentBatch.Put(key, value); } } else { if (value == null) { _db.Remove(key); } else { _db.Put(key, value); } } } }
public void Put <T>(string key, T item) { using (var objectStream = new MemoryStream()) { _formatter.Serialize(objectStream, item); _dbInstance.Put(Encoding.UTF8.GetBytes(key), objectStream.GetBuffer()); } }
/// <inheritdoc/> public override void StageTransactionIds(IImmutableSet <TxId> txids) { foreach (TxId txId in txids) { byte[] key = StagedTxKey(txId); _stagedTxDb.Put(key, EmptyBytes); } }
public static void Put(this RocksDb db, byte table, byte[] key, byte[] value) { Span <byte> dbkey = stackalloc byte[key.Length + 1]; dbkey[0] = table; key.AsSpan().CopyTo(dbkey.Slice(1)); db.Put(dbkey.ToArray(), value); }
public IActionResult LoadESElectionConfiguration() { if (!HttpContext.Session.Keys.Contains(IsAuthenticated)) { return(RedirectToAction("Signin")); } var confAPI = new VotingSystemConfiguration(); lock (_conf) { if (_conf.Get(ESElectionConfigurationKey) != null) { return(new ForbidResult()); } var v = _conf.Get(APIConfigurationKey); if (v != null) { confAPI = VotingSystemConfiguration.FromJson(v); } } var urlBuilder = new System.Text.StringBuilder(); urlBuilder.Append(confAPI.ElectionSystemAPI.TrimEnd('/')).Append("/RunningElections"); var req = WebRequest.Create(urlBuilder.ToString()); var resp = req.GetResponse(); var pdata = (new StreamReader(resp.GetResponseStream())).ReadToEnd(); var dp = dataProtector.CreateProtector("EligereMetadataExchange"); var data = dp.Unprotect(pdata); var fn = Path.Combine(contentRootPath, $"wwwroot/temp/log.txt"); System.IO.File.WriteAllText(fn, data); var eldesc = JsonSerializer.Deserialize <ElectionGuard.ElectionDescription>(data); lock (_conf) { _conf.Put(ESElectionConfigurationKey, JsonSerializer.Serialize <ElectionGuard.ElectionDescription>(eldesc)); } return(View("ShowESElectionConfiguration", eldesc)); }
private static void RocksDb1() { string directory = @"C:\Users\Peska\source\repos\Esent.Tests\RocksDB"; int maxSize = 1_000_000; TestObject[] objects = new TestObject[maxSize]; for (int i = 0; i < maxSize; i++) { objects[i] = TestObject.Create(); if (i % 1000 == 0) { Console.WriteLine($"Created {i} records"); } } DbOptions options = new DbOptions().SetCreateIfMissing(); using (RocksDb rocksDb = RocksDbSharp.RocksDb.Open(options, directory)) { for (int i = 0; i < maxSize; i++) { string json = JsonConvert.SerializeObject(objects[i]); rocksDb.Put(objects[i].EQNum, json); if (i % 1000 == 0) { Console.WriteLine($"Indexed {i} records"); } } Random random = new Random(); for (int i = 0; i < maxSize; i++) { int randomInt = random.Next(maxSize - 1); TestObject testObject = objects[randomInt]; TestObject fromDict = JsonConvert.DeserializeObject <TestObject>(rocksDb.Get(testObject.EQNum)); if (testObject.EQNum != fromDict.EQNum) { throw new Exception("What the f**k"); } if (i % 1000 == 0) { Console.WriteLine($"Retrieved {i} records"); } } Iterator iterator = rocksDb.NewIterator(); } }
/// <summary> /// Update or Insert Key's value from Rocksdb /// </summary> /// <param name="Key">Search Key</param> /// <returns></returns> bool PutDb(string Key, string Value) { try { db.Put(Key, Value); return(true); } catch (Exception exc) { Log.Error(exc, $"PutDb(key:{Key}, value:{Value})"); return(false); } }
/// <inheritdoc/> public override void DeleteChainId(Guid chainId) { ColumnFamilyHandle cf = GetColumnFamily(_chainDb, chainId); if (HasFork(chainId)) { _chainDb.Put(DeletedKey, new byte[0], cf); // We need only chain indexes, not tx nonces at this time because they already had // been copied on .ForkTxNonces(). // FIXME: We should remove this code after adjusting .ForkTxNonces(). using var batch = new WriteBatch(); foreach (Iterator k in IterateDb(_chainDb, TxNonceKeyPrefix, chainId)) { batch.Delete(k.Key(), cf); } _chainDb.Write(batch); return; } _logger.Debug($"Deleting chainID: {chainId}."); Guid?prevChain = GetPreviousChainInfo(cf)?.Item1; string cfName = chainId.ToString(); try { _chainDb.DropColumnFamily(cfName); if (prevChain is { } prevChainNotNull) { lock (_chainForkDeleteLock) { if (HasFork(prevChainNotNull)) { ColumnFamilyHandle prevCf = GetColumnFamily(_chainDb, prevChainNotNull); RemoveFork(prevCf, chainId); if (IsDeletionMarked(prevCf)) { DeleteChainId(prevChainNotNull); } } } } } catch (KeyNotFoundException) { // Do nothing according to the specification: DeleteChainId() should be idempotent. _logger.Debug($"No such chain ID in _chainDb: {cfName}.", cfName); } catch (Exception e) { LogUnexpectedException(nameof(DeleteChainId), e); } }
public string[] ElectionGuardGenGuardians(int numGuardians, int quorum) { // ElectionGuard generation ElectionGuard.ElectionDescription eldesc; lock (_conf) { if (_conf.Get(ESElectionConfigurationKey) == null) { throw new Exception("ESConfiguration missing"); } eldesc = JsonSerializer.Deserialize <ElectionGuard.ElectionDescription>(_conf.Get(ESElectionConfigurationKey)); } var guardianIds = new string[numGuardians]; for (var i = 0; i < guardianIds.Length; i++) { guardianIds[i] = $"Guardian-{i}"; } var guardians = new List <ElectionGuard.Guardian>(); for (var i = 0; i < numGuardians; i++) { guardians.Add(GuardianApi.Guardian(null, null, guardianIds[i], numGuardians, quorum, i)); } var guardianPubKeys = guardians.ConvertAll(g => g.election_key_pair.public_key).ToArray(); var electionJointKey = MediatorApi.ElectionCombine(guardianPubKeys); var electionContext = MediatorApi.ElectionContext(eldesc, electionJointKey.joint_key, numGuardians, quorum); // Persists only the public part lock (egSecureBallot) { egSecureBallot.Put("GuardianPubKeys", JsonSerializer.Serialize(guardianPubKeys)); egSecureBallot.Put("ElectionJointKey", JsonSerializer.Serialize(electionJointKey)); egSecureBallot.Put("ElectionContext", JsonSerializer.Serialize(electionContext)); egSecureBallot.Put("Nonce", "110191403412906482859082647039385408757148325819889522238592336039604240167009"); egSecureBallot.Put("SeedHash", "110191403412906482859082647039385908787142225838889522238592336039604240167009"); } return(guardians.ConvertAll(g => JsonSerializer.Serialize(g)).ToArray()); }
/// <inheritdoc/> public override void SetBlockPerceivedTime( HashDigest <SHA256> blockHash, DateTimeOffset perceivedTime ) { byte[] key = BlockKey(blockHash); _blockPerceptionDb.Put( key, NetworkOrderBitsConverter.GetBytes(perceivedTime.ToUnixTimeMilliseconds()) ); }
private static void WriteHashCache(AbsolutePath file, Hash hash) { using var ms = new MemoryStream(20); using var bw = new BinaryWriter(ms); bw.Write(HashCacheVersion); var lastModified = file.LastModifiedUtc.AsUnixTime(); bw.Write(lastModified); bw.Write((ulong)hash); _hashCache.Put(Encoding.UTF8.GetBytes(file.Normalize()), ms.ToArray()); }
public void create_rocksdb_test() { Parallel.ForEach(Enumerable.Range(1, 10), i => { // Using strings below, but can also use byte arrays for both keys and values // much care has been taken to minimize buffer copying _rocksDb.Put($"key{i}", "value"); string value = _rocksDb.Get($"key{i}"); Assert.AreEqual("value", value); _rocksDb.Remove($"key{i}"); }); _rocksDb.Dispose(); }
/// <inheritdoc/> public override void PutBlock <T>(Block <T> block) { if (_blockCache.ContainsKey(block.Hash)) { return; } byte[] key = BlockKey(block.Hash); if (!(_blockIndexDb.Get(key) is null)) { return; } long timestamp = block.Timestamp.ToUnixTimeSeconds(); foreach (Transaction <T> tx in block.Transactions) { PutTransaction(tx); } _rwBlockLock.EnterWriteLock(); try { string blockDbName = $"epoch{timestamp / _blockEpochUnitSeconds}"; RocksDb blockDb; lock (_blockDbCache) { if (!_blockDbCache.TryGetValue(blockDbName, out blockDb)) { blockDb = RocksDBUtils.OpenRocksDb(_options, BlockDbPath(blockDbName)); _blockDbCache.AddOrUpdate(blockDbName, blockDb); } } BlockDigest digest = BlockDigest.FromBlock(block); byte[] value = digest.Serialize(); blockDb.Put(key, value); _blockIndexDb.Put(key, RocksDBStoreBitConverter.GetBytes(blockDbName)); _blockCache.AddOrUpdate(block.Hash, digest); } catch (Exception e) { LogUnexpectedException(nameof(PutBlock), e); } finally { _rwBlockLock.ExitWriteLock(); } }