public void BuildFilterAndMatchValues()
{
var names = from name in new[] { "New York", "Amsterdam", "Paris", "Buenos Aires", "La Habana" }
select Encoding.ASCII.GetBytes(name);
var key = new byte[] { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 };
var filter = GolombRiceFilter.Build(key, names, 0x10);
// The filter should match all ther values that were added.
foreach (var name in names)
{
Assert.True(filter.Match(name, key));
}
// The filter should NOT match any extra value.
Assert.False(filter.Match(Encoding.ASCII.GetBytes("Porto Alegre"), key));
Assert.False(filter.Match(Encoding.ASCII.GetBytes("Madrid"), key));
// The filter should match because it has one element indexed: Buenos Aires.
var otherCities = new[] { "La Paz", "Barcelona", "El Cairo", "Buenos Aires", "Asunción" };
var otherNames = from name in otherCities select Encoding.ASCII.GetBytes(name);
Assert.True(filter.MatchAny(otherNames, key));
// The filter should NOT match because it doesn't have any element indexed.
var otherCities2 = new[] { "La Paz", "Barcelona", "El Cairo", "Córdoba", "Asunción" };
var otherNames2 = from name in otherCities2 select Encoding.ASCII.GetBytes(name);
Assert.False(filter.MatchAny(otherNames2, key));
}
public GolombRiceFilter Build(Block block)
{
var key = block.GetHash().ToBytes();
var buffer = new List <byte[]>
{
key
};
foreach (var tx in block.Transactions)
{
foreach (var txOutput in tx.Outputs)
{
var isValidPayToWitness = P2wpkh.CheckScriptPubKey(txOutput.ScriptPubKey);
if (isValidPayToWitness)
{
var witKeyId = P2wpkh.ExtractScriptPubKeyParameters(txOutput.ScriptPubKey);
buffer.Add(witKeyId.ToBytes());
}
}
}
return(GolombRiceFilter.Build(key, buffer, P));
}
public void CreateStoreTest()
{
const byte P = 20;
const int blockCount = 100;
const int maxBlockSize = 4 * 1000 * 1000;
const int avgTxSize = 250; // Currently the average is around 1kb.
const int txoutCountPerBlock = maxBlockSize / avgTxSize;
const int avgTxoutPushDataSize = 20; // P2PKH scripts has 20 bytes.
var key = new byte[] { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 };
// Generation of data to be added into the filter
var random = new Random();
var dataDirectory = new DirectoryInfo(Path.Combine(SharedFixture.DataDir, nameof(CreateStoreTest)));
if (dataDirectory.Exists)
{
foreach (var fileInfo in dataDirectory.GetFiles())
{
fileInfo.Delete();
}
}
var blocks = new List <GolombRiceFilter>(blockCount);
using (var repo = GcsFilterRepository.Open(Path.Combine(SharedFixture.DataDir, nameof(CreateStoreTest))))
{
for (var i = 0; i < blockCount; i++)
{
var txouts = new List <byte[]>(txoutCountPerBlock);
for (var j = 0; j < txoutCountPerBlock; j++)
{
var pushDataBuffer = new byte[avgTxoutPushDataSize];
random.NextBytes(pushDataBuffer);
txouts.Add(pushDataBuffer);
}
var filter = GolombRiceFilter.Build(key, txouts, P);
blocks.Add(filter);
repo.Put(Hashes.Hash256(filter.Data.ToByteArray()), filter);
}
}
using (var repo = GcsFilterRepository.Open(Path.Combine(SharedFixture.DataDir, nameof(CreateStoreTest))))
{
var blockIndexes = Enumerable.Range(0, blockCount).ToList();
blockIndexes.Shuffle();
foreach (var blkIndx in blockIndexes)
{
var block = blocks[blkIndx];
var blockFilter = block;
var blockFilterId = Hashes.Hash256(blockFilter.Data.ToByteArray());
var savedFilter = repo.Get(blockFilterId);
var savedFilterId = Hashes.Hash256(savedFilter.Data.ToByteArray());
Assert.Equal(blockFilterId, savedFilterId);
}
}
}
public void FalsePositivesTest()
{
// Given this library can be used for building and query filters for each block of
// the bitcoin's blockchain, we must be sure it performs well, specially in the queries.
// Considering a 4MB block (overestimated) with an average transaction size of 250 bytes (underestimated)
// gives us 16000 transactions (this is about 27 tx/sec). Assuming 2.5 txouts per tx we have 83885 txouts
// per block.
const byte P = 20;
const int blockCount = 100;
const int maxBlockSize = 4 * 1000 * 1000;
const int avgTxSize = 250; // Currently the average is around 1kb.
const int txoutCountPerBlock = maxBlockSize / avgTxSize;
const int avgTxoutPushDataSize = 20; // P2PKH scripts has 20 bytes.
const int walletAddressCount = 1000; // We estimate that our user will have 1000 addresses.
var key = new byte[] { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 };
// Generation of data to be added into the filter
var random = new Random();
var sw = new Stopwatch();
var blocks = new List <BlockFilter>(blockCount);
for (var i = 0; i < blockCount; i++)
{
var txouts = new List <byte[]>(txoutCountPerBlock);
for (var j = 0; j < txoutCountPerBlock; j++)
{
var pushDataBuffer = new byte[avgTxoutPushDataSize];
random.NextBytes(pushDataBuffer);
txouts.Add(pushDataBuffer);
}
sw.Start();
var filter = GolombRiceFilter.Build(key, txouts, P);
sw.Stop();
blocks.Add(new BlockFilter(filter, txouts));
}
sw.Reset();
var walletAddresses = new List <byte[]>(walletAddressCount);
var falsePositiveCount = 0;
for (var i = 0; i < walletAddressCount; i++)
{
var walletAddress = new byte[avgTxoutPushDataSize];
random.NextBytes(walletAddress);
walletAddresses.Add(walletAddress);
}
sw.Start();
// Check that the filter can match every single txout in every block.
foreach (var block in blocks)
{
if (block.Filter.MatchAny(walletAddresses, key))
{
falsePositiveCount++;
}
}
sw.Stop();
Assert.True(falsePositiveCount < 5);
// Filter has to mat existing values
sw.Start();
var falseNegativeCount = 0;
// Check that the filter can match every single txout in every block.
foreach (var block in blocks)
{
if (!block.Filter.MatchAny(block.Data, key))
{
falseNegativeCount++;
}
}
sw.Stop();
Assert.Equal(0, falseNegativeCount);
}
public void Synchronize()
{
Interlocked.Exchange(ref _running, 1);
Task.Run(async() =>
{
try
{
var blockCount = await RpcClient.GetBlockCountAsync();
var isIIB = true; // Initial Index Building phase
while (IsRunning)
{
try
{
// If stop was requested return.
if (IsRunning == false)
{
return;
}
var height = StartingHeight;
uint256 prevHash = null;
using (await IndexLock.LockAsync())
{
if (Index.Count != 0)
{
var lastIndex = Index.Last();
height = lastIndex.BlockHeight + 1;
prevHash = lastIndex.BlockHash;
}
}
if (blockCount - (int)height <= 100)
{
isIIB = false;
}
Block block = null;
try
{
block = await RpcClient.GetBlockAsync(height);
}
catch (RPCException) // if the block didn't come yet
{
await Task.Delay(1000);
continue;
}
if (prevHash != null)
{
// In case of reorg:
if (prevHash != block.Header.HashPrevBlock && !isIIB) // There is no reorg in IIB
{
Logger.LogInfo <IndexBuilderService>($"REORG Invalid Block: {prevHash}");
// 1. Rollback index
using (await IndexLock.LockAsync())
{
Index.RemoveLast();
}
// 2. Serialize Index. (Remove last line.)
var lines = File.ReadAllLines(IndexFilePath);
File.WriteAllLines(IndexFilePath, lines.Take(lines.Length - 1).ToArray());
// 3. Rollback Bech32UtxoSet
if (Bech32UtxoSetHistory.Count != 0)
{
Bech32UtxoSetHistory.Last().Rollback(Bech32UtxoSet); // The Bech32UtxoSet MUST be recovered to its previous state.
Bech32UtxoSetHistory.RemoveLast();
// 4. Serialize Bech32UtxoSet.
await File.WriteAllLinesAsync(Bech32UtxoSetFilePath, Bech32UtxoSet
.Select(entry => entry.Key.Hash + ":" + entry.Key.N + ":" + ByteHelpers.ToHex(entry.Value.ToCompressedBytes())));
}
// 5. Skip the current block.
continue;
}
}
if (!isIIB)
{
if (Bech32UtxoSetHistory.Count >= 100)
{
Bech32UtxoSetHistory.RemoveFirst();
}
Bech32UtxoSetHistory.Add(new ActionHistoryHelper());
}
var scripts = new HashSet <Script>();
foreach (var tx in block.Transactions)
{
for (int i = 0; i < tx.Outputs.Count; i++)
{
var output = tx.Outputs[i];
if (!output.ScriptPubKey.IsPayToScriptHash && output.ScriptPubKey.IsWitness)
{
var outpoint = new OutPoint(tx.GetHash(), i);
Bech32UtxoSet.Add(outpoint, output.ScriptPubKey);
if (!isIIB)
{
Bech32UtxoSetHistory.Last().StoreAction(ActionHistoryHelper.Operation.Add, outpoint, output.ScriptPubKey);
}
scripts.Add(output.ScriptPubKey);
}
}
foreach (var input in tx.Inputs)
{
var found = Bech32UtxoSet.SingleOrDefault(x => x.Key == input.PrevOut);
if (found.Key != default)
{
Script val = Bech32UtxoSet[input.PrevOut];
Bech32UtxoSet.Remove(input.PrevOut);
if (!isIIB)
{
Bech32UtxoSetHistory.Last().StoreAction(ActionHistoryHelper.Operation.Remove, input.PrevOut, val);
}
scripts.Add(found.Value);
}
}
}
// https://github.com/bitcoin/bips/blob/master/bip-0158.mediawiki
// The parameter k MUST be set to the first 16 bytes of the hash of the block for which the filter
// is constructed.This ensures the key is deterministic while still varying from block to block.
var key = block.GetHash().ToBytes().Take(16).ToArray();
GolombRiceFilter filter = null;
if (scripts.Count != 0)
{
filter = GolombRiceFilter.Build(key, scripts.Select(x => x.ToCompressedBytes()));
}
var filterModel = new FilterModel
{
BlockHash = block.GetHash(),
BlockHeight = height,
Filter = filter
};
await File.AppendAllLinesAsync(IndexFilePath, new[] { filterModel.ToLine() });
using (await IndexLock.LockAsync())
{
Index.Add(filterModel);
}
if (File.Exists(Bech32UtxoSetFilePath))
{
File.Delete(Bech32UtxoSetFilePath);
}
await File.WriteAllLinesAsync(Bech32UtxoSetFilePath, Bech32UtxoSet
.Select(entry => entry.Key.Hash + ":" + entry.Key.N + ":" + ByteHelpers.ToHex(entry.Value.ToCompressedBytes())));
if (blockCount - height <= 3 || height % 100 == 0) // If not close to the tip, just log debug.
{
Logger.LogInfo <IndexBuilderService>($"Created filter for block: {height}.");
}
else
{
Logger.LogDebug <IndexBuilderService>($"Created filter for block: {height}.");
}
}
catch (Exception ex)
{
Logger.LogDebug <IndexBuilderService>(ex);
}
}
}
finally
{
if (IsStopping)
{
Interlocked.Exchange(ref _running, 3);
}
}
});
}
