public void BuildFilterAndMatchValues() { var names = from name in new[] { "New York", "Amsterdam", "Paris", "Buenos Aires", "La Habana" } select Encoding.ASCII.GetBytes(name); var key = new byte[] { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 }; var filter = GolombRiceFilter.Build(key, names, 0x10); // The filter should match all ther values that were added. foreach (var name in names) { Assert.True(filter.Match(name, key)); } // The filter should NOT match any extra value. Assert.False(filter.Match(Encoding.ASCII.GetBytes("Porto Alegre"), key)); Assert.False(filter.Match(Encoding.ASCII.GetBytes("Madrid"), key)); // The filter should match because it has one element indexed: Buenos Aires. var otherCities = new[] { "La Paz", "Barcelona", "El Cairo", "Buenos Aires", "Asunción" }; var otherNames = from name in otherCities select Encoding.ASCII.GetBytes(name); Assert.True(filter.MatchAny(otherNames, key)); // The filter should NOT match because it doesn't have any element indexed. var otherCities2 = new[] { "La Paz", "Barcelona", "El Cairo", "Córdoba", "Asunción" }; var otherNames2 = from name in otherCities2 select Encoding.ASCII.GetBytes(name); Assert.False(filter.MatchAny(otherNames2, key)); }
public GolombRiceFilter Build(Block block) { var key = block.GetHash().ToBytes(); var buffer = new List <byte[]> { key }; foreach (var tx in block.Transactions) { foreach (var txOutput in tx.Outputs) { var isValidPayToWitness = P2wpkh.CheckScriptPubKey(txOutput.ScriptPubKey); if (isValidPayToWitness) { var witKeyId = P2wpkh.ExtractScriptPubKeyParameters(txOutput.ScriptPubKey); buffer.Add(witKeyId.ToBytes()); } } } return(GolombRiceFilter.Build(key, buffer, P)); }
public void CreateStoreTest() { const byte P = 20; const int blockCount = 100; const int maxBlockSize = 4 * 1000 * 1000; const int avgTxSize = 250; // Currently the average is around 1kb. const int txoutCountPerBlock = maxBlockSize / avgTxSize; const int avgTxoutPushDataSize = 20; // P2PKH scripts has 20 bytes. var key = new byte[] { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 }; // Generation of data to be added into the filter var random = new Random(); var dataDirectory = new DirectoryInfo(Path.Combine(SharedFixture.DataDir, nameof(CreateStoreTest))); if (dataDirectory.Exists) { foreach (var fileInfo in dataDirectory.GetFiles()) { fileInfo.Delete(); } } var blocks = new List <GolombRiceFilter>(blockCount); using (var repo = GcsFilterRepository.Open(Path.Combine(SharedFixture.DataDir, nameof(CreateStoreTest)))) { for (var i = 0; i < blockCount; i++) { var txouts = new List <byte[]>(txoutCountPerBlock); for (var j = 0; j < txoutCountPerBlock; j++) { var pushDataBuffer = new byte[avgTxoutPushDataSize]; random.NextBytes(pushDataBuffer); txouts.Add(pushDataBuffer); } var filter = GolombRiceFilter.Build(key, txouts, P); blocks.Add(filter); repo.Put(Hashes.Hash256(filter.Data.ToByteArray()), filter); } } using (var repo = GcsFilterRepository.Open(Path.Combine(SharedFixture.DataDir, nameof(CreateStoreTest)))) { var blockIndexes = Enumerable.Range(0, blockCount).ToList(); blockIndexes.Shuffle(); foreach (var blkIndx in blockIndexes) { var block = blocks[blkIndx]; var blockFilter = block; var blockFilterId = Hashes.Hash256(blockFilter.Data.ToByteArray()); var savedFilter = repo.Get(blockFilterId); var savedFilterId = Hashes.Hash256(savedFilter.Data.ToByteArray()); Assert.Equal(blockFilterId, savedFilterId); } } }
public void FalsePositivesTest() { // Given this library can be used for building and query filters for each block of // the bitcoin's blockchain, we must be sure it performs well, specially in the queries. // Considering a 4MB block (overestimated) with an average transaction size of 250 bytes (underestimated) // gives us 16000 transactions (this is about 27 tx/sec). Assuming 2.5 txouts per tx we have 83885 txouts // per block. const byte P = 20; const int blockCount = 100; const int maxBlockSize = 4 * 1000 * 1000; const int avgTxSize = 250; // Currently the average is around 1kb. const int txoutCountPerBlock = maxBlockSize / avgTxSize; const int avgTxoutPushDataSize = 20; // P2PKH scripts has 20 bytes. const int walletAddressCount = 1000; // We estimate that our user will have 1000 addresses. var key = new byte[] { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 }; // Generation of data to be added into the filter var random = new Random(); var sw = new Stopwatch(); var blocks = new List <BlockFilter>(blockCount); for (var i = 0; i < blockCount; i++) { var txouts = new List <byte[]>(txoutCountPerBlock); for (var j = 0; j < txoutCountPerBlock; j++) { var pushDataBuffer = new byte[avgTxoutPushDataSize]; random.NextBytes(pushDataBuffer); txouts.Add(pushDataBuffer); } sw.Start(); var filter = GolombRiceFilter.Build(key, txouts, P); sw.Stop(); blocks.Add(new BlockFilter(filter, txouts)); } sw.Reset(); var walletAddresses = new List <byte[]>(walletAddressCount); var falsePositiveCount = 0; for (var i = 0; i < walletAddressCount; i++) { var walletAddress = new byte[avgTxoutPushDataSize]; random.NextBytes(walletAddress); walletAddresses.Add(walletAddress); } sw.Start(); // Check that the filter can match every single txout in every block. foreach (var block in blocks) { if (block.Filter.MatchAny(walletAddresses, key)) { falsePositiveCount++; } } sw.Stop(); Assert.True(falsePositiveCount < 5); // Filter has to mat existing values sw.Start(); var falseNegativeCount = 0; // Check that the filter can match every single txout in every block. foreach (var block in blocks) { if (!block.Filter.MatchAny(block.Data, key)) { falseNegativeCount++; } } sw.Stop(); Assert.Equal(0, falseNegativeCount); }
public void Synchronize() { Interlocked.Exchange(ref _running, 1); Task.Run(async() => { try { var blockCount = await RpcClient.GetBlockCountAsync(); var isIIB = true; // Initial Index Building phase while (IsRunning) { try { // If stop was requested return. if (IsRunning == false) { return; } var height = StartingHeight; uint256 prevHash = null; using (await IndexLock.LockAsync()) { if (Index.Count != 0) { var lastIndex = Index.Last(); height = lastIndex.BlockHeight + 1; prevHash = lastIndex.BlockHash; } } if (blockCount - (int)height <= 100) { isIIB = false; } Block block = null; try { block = await RpcClient.GetBlockAsync(height); } catch (RPCException) // if the block didn't come yet { await Task.Delay(1000); continue; } if (prevHash != null) { // In case of reorg: if (prevHash != block.Header.HashPrevBlock && !isIIB) // There is no reorg in IIB { Logger.LogInfo <IndexBuilderService>($"REORG Invalid Block: {prevHash}"); // 1. Rollback index using (await IndexLock.LockAsync()) { Index.RemoveLast(); } // 2. Serialize Index. (Remove last line.) var lines = File.ReadAllLines(IndexFilePath); File.WriteAllLines(IndexFilePath, lines.Take(lines.Length - 1).ToArray()); // 3. Rollback Bech32UtxoSet if (Bech32UtxoSetHistory.Count != 0) { Bech32UtxoSetHistory.Last().Rollback(Bech32UtxoSet); // The Bech32UtxoSet MUST be recovered to its previous state. Bech32UtxoSetHistory.RemoveLast(); // 4. Serialize Bech32UtxoSet. await File.WriteAllLinesAsync(Bech32UtxoSetFilePath, Bech32UtxoSet .Select(entry => entry.Key.Hash + ":" + entry.Key.N + ":" + ByteHelpers.ToHex(entry.Value.ToCompressedBytes()))); } // 5. Skip the current block. continue; } } if (!isIIB) { if (Bech32UtxoSetHistory.Count >= 100) { Bech32UtxoSetHistory.RemoveFirst(); } Bech32UtxoSetHistory.Add(new ActionHistoryHelper()); } var scripts = new HashSet <Script>(); foreach (var tx in block.Transactions) { for (int i = 0; i < tx.Outputs.Count; i++) { var output = tx.Outputs[i]; if (!output.ScriptPubKey.IsPayToScriptHash && output.ScriptPubKey.IsWitness) { var outpoint = new OutPoint(tx.GetHash(), i); Bech32UtxoSet.Add(outpoint, output.ScriptPubKey); if (!isIIB) { Bech32UtxoSetHistory.Last().StoreAction(ActionHistoryHelper.Operation.Add, outpoint, output.ScriptPubKey); } scripts.Add(output.ScriptPubKey); } } foreach (var input in tx.Inputs) { var found = Bech32UtxoSet.SingleOrDefault(x => x.Key == input.PrevOut); if (found.Key != default) { Script val = Bech32UtxoSet[input.PrevOut]; Bech32UtxoSet.Remove(input.PrevOut); if (!isIIB) { Bech32UtxoSetHistory.Last().StoreAction(ActionHistoryHelper.Operation.Remove, input.PrevOut, val); } scripts.Add(found.Value); } } } // https://github.com/bitcoin/bips/blob/master/bip-0158.mediawiki // The parameter k MUST be set to the first 16 bytes of the hash of the block for which the filter // is constructed.This ensures the key is deterministic while still varying from block to block. var key = block.GetHash().ToBytes().Take(16).ToArray(); GolombRiceFilter filter = null; if (scripts.Count != 0) { filter = GolombRiceFilter.Build(key, scripts.Select(x => x.ToCompressedBytes())); } var filterModel = new FilterModel { BlockHash = block.GetHash(), BlockHeight = height, Filter = filter }; await File.AppendAllLinesAsync(IndexFilePath, new[] { filterModel.ToLine() }); using (await IndexLock.LockAsync()) { Index.Add(filterModel); } if (File.Exists(Bech32UtxoSetFilePath)) { File.Delete(Bech32UtxoSetFilePath); } await File.WriteAllLinesAsync(Bech32UtxoSetFilePath, Bech32UtxoSet .Select(entry => entry.Key.Hash + ":" + entry.Key.N + ":" + ByteHelpers.ToHex(entry.Value.ToCompressedBytes()))); if (blockCount - height <= 3 || height % 100 == 0) // If not close to the tip, just log debug. { Logger.LogInfo <IndexBuilderService>($"Created filter for block: {height}."); } else { Logger.LogDebug <IndexBuilderService>($"Created filter for block: {height}."); } } catch (Exception ex) { Logger.LogDebug <IndexBuilderService>(ex); } } } finally { if (IsStopping) { Interlocked.Exchange(ref _running, 3); } } }); }