private async Task DeleteIfDeprecatedAsync(DigestableSafeMutexIoManager ioManager)
{
string firstLine;
using (var content = ioManager.OpenText())
{
firstLine = await content.ReadLineAsync().ConfigureAwait(false);
}
try
{
FilterModel.FromLine(firstLine);
}
catch
{
Logger.LogWarning("Old Index file detected. Deleting it.");
MatureIndexFileManager.DeleteMe();
ImmatureIndexFileManager.DeleteMe();
Logger.LogWarning("Successfully deleted old Index file.");
}
}
public static FilterModel GetStartingFilter(Network network)
{
var startingHeader = SmartHeader.GetStartingHeader(network);
if (network == Network.Main)
{
return(FilterModel.FromLine($"{startingHeader.Height}:{startingHeader.BlockHash}:02832810ec08a0:{startingHeader.PrevHash}:{startingHeader.BlockTime.ToUnixTimeSeconds()}"));
}
else if (network == Network.TestNet)
{
return(FilterModel.FromLine($"{startingHeader.Height}:{startingHeader.BlockHash}:00000000000f0d5edcaeba823db17f366be49a80d91d15b77747c2e017b8c20a:{startingHeader.PrevHash}:{startingHeader.BlockTime.ToUnixTimeSeconds()}"));
}
else if (network == Network.RegTest)
{
GolombRiceFilter filter = IndexBuilderService.CreateDummyEmptyFilter(startingHeader.BlockHash);
return(FilterModel.FromLine($"{startingHeader.Height}:{startingHeader.BlockHash}:{filter.ToString()}:{startingHeader.PrevHash}:{startingHeader.BlockTime.ToUnixTimeSeconds()}"));
}
else
{
throw new NotSupportedNetworkException(network);
}
}
public static FilterModel GetStartingFilter(Network network)
{
var startingHeader = SmartHeader.GetStartingHeader(network);
if (network == Network.Main)
{
return(FilterModel.FromLine($"{startingHeader.Height}:{startingHeader.BlockHash}:fd37025542ac13565ae0612c2d2c178df07d05402c527fd91942a68300e20853315e278ebc861f7ec7e20ee5c7ea39197ab66ec38c6cacfb7a960e87eafe739fbb58e703811b2ab0cc9ac55c34f3a6d70536776a34679680d5539e3ff089a08d6fed732e3c26e4b5527790cd14a7bd83c8142b5cf4f10fbd09baed653cb4ebf9100bcc5d30878b1aaa01a00230d505004e5d686fd752ec8624f88d640867cab753ca1d3b7829b05a2c20606279242688ba319c2173a5a885399b58c45127bdf976c4d45e79425ff809da944d30405f929c10186e1666bb444cdeece71133049e5e44a265d44c4122e83919d6573220f3b9bdf38ee3037e91b0ae0485707e9a8755dd530fff74338652b55aa547f3c505db5f0a1b92f95fd95852d48167ef857d55500dfb8c7209f05c1669a73f15e7bb26537462a05da4f0d803315dd43630b23019be48a556ebe50752bdd37080460254e4d6d92be80ab0ed8efd882fd0c360fbf00b387b6e5ef4d9a27267b2f51151684fa7cf3064f8e2fc918843215512c838fdd35270da33df3d203e02539f1b030863619a4fdd93b7771de31d35436a50c904225942fda79e2606a21d64dbb110c79f202d10acd94a2ac7cf24cff2dc816ac6828941abec3d9ae721c09f5a2070d3a99aba0fb1e33d7de26c3c2b7f40b002719138826a670c38d56fe5a67e827bf48865f2c34e4eafa68486ecb419118144d212d5a10ee41f26c07b2456c5a0b74fb86cb4eaec27ac2cf80847e57d711754589716c70eea0d1c595b6dd413aa588542acc5ea9d5c102f3b4aac876bb91f57c1eea063d520517bd0c48e2d79ec2164525e1c24a34e8f50e4256790395fcdd5c71b50949738ab92d44476c2073e16bedb4b31517d2264ae4d824b0990fe316919c432edc9f71c1a6a21937786b68e9e11f788fba04ad1253cfcd3148698ec8a4a1054a33aa51adcdb61c922d44ebefb6dc1aab67d07b48bb702606047230e4439bec8d1b371683be41617e4f163509b09f5cd6577956b1857a338764486017949f594c90be92783776ff0ba0d6d517d18d5842c36eee044290df2ca0ad3818f3bcd874ad23e41a77c441da273f20e3472801a4892b635d3d2a0b76ca1c0025b5283850897993410f5c654527ebfc446445ca10d0264c964b86719a108179f303165b3b5f53441d9fe9f6e1a1ae2d2d61c6c34752ca9b02b6c8fb2166dba95370ad1cace915e90cab14797a33be4454fe44dfecb015c59e76b60336192b262169db6461b26da3b03cdcca4c185e7d76bef944ea02a350449e9119aeb82ac094b87cd3fec4bd289c55bf008e3eb614017240c73fb3440de1cbd0f1e634bd997f1c10ccf97598b8d0144bd8608999a6420ff65cfb17f275b33e16d08bd5f01ff2a98b02f3cc84b1931b68b4d47ebab850824d55029e2deaab509bbe09b698675add8a5119630207c41c54a9503ea2ab15ce4446504ce19c52313733a95fed0dea0f9e56c5658d52d43f09b7a246140dac1e49aeb00d7d0b50aac0fab135249f523ee1fc59d8d2c18af69db23fc26a006451e132448bc7b671fa864897a8e23b1bf4b97397d091f8106e12174efd82e7f4479aab210f488cd4cb45a093392ecab407ad3a8836c771ec87d36d41aeeac20c36b3d39c87ea1eaba86411a78ca33f658cceadb903ec7482a395f11b7aa54eecd9da1a9a542695723fff6056ac736b788abd2498488a6737236d70e7aceecdc71e4878c2f1e390866020cf20681230f3858dd7ae44b9bdb0a3726b12db3b42b2863cf03d3e34b733d9c63e92c5b07bf942dc740a2c902621c1f74c4e00e3a36002fb8a24febc89aa5350544566de06ba81aa915e115368aabe7fdedb676082abac78a1b216d7e56c8ed0da8ada71bc2f46ade811233f9568308136700e21b5941286d0d48fe43009b48161837e5255e639d8a4225928a2637d745cbc580c9b14c50860dba2f1be321463e47cb4914c81c7b81206090aa23a73103f139dfca303447cc7140eef9a5f5235675073cdcd5635abad96de169975f2cf06689c9dc9c9d7557d504c7ec1541b493769a2106537d56d489f8800d5d76870c716678ff3672c76b8a179b0a601858b8973ed59a9ecc13f115b44d2a3251e658b95bd5f3f1aa21c57a6eb960dcbf968890:{startingHeader.PrevHash}:{startingHeader.BlockTime.ToUnixTimeSeconds()}"));
}
else if (network == Network.TestNet)
{
return(FilterModel.FromLine($"{startingHeader.Height}:{startingHeader.BlockHash}:00000000000f0d5edcaeba823db17f366be49a80d91d15b77747c2e017b8c20a:{startingHeader.PrevHash}:{startingHeader.BlockTime.ToUnixTimeSeconds()}"));
}
else if (network == Network.RegTest)
{
GolombRiceFilter filter = IndexBuilderService.CreateDummyEmptyFilter(startingHeader.BlockHash);
return(FilterModel.FromLine($"{startingHeader.Height}:{startingHeader.BlockHash}:{filter}:{startingHeader.PrevHash}:{startingHeader.BlockTime.ToUnixTimeSeconds()}"));
}
else
{
throw new NotSupportedNetworkException(network);
}
}
public static FilterModel GetStartingFilter(Network network)
{
var startingHeader = SmartHeader.GetStartingHeader(network);
if (network == NBitcoin.Altcoins.Litecoin.Instance.Mainnet)
{
return(FilterModel.FromLine($"{startingHeader.Height}:{startingHeader.BlockHash}:{startingHeader.BlockHash}:{startingHeader.PrevHash}:{startingHeader.BlockTime.ToUnixTimeSeconds()}"));
}
else if (network == NBitcoin.Altcoins.Litecoin.Instance.Testnet)
{
return(FilterModel.FromLine($"{startingHeader.Height}:{startingHeader.BlockHash}:{startingHeader.BlockHash}:{startingHeader.PrevHash}:{startingHeader.BlockTime.ToUnixTimeSeconds()}"));
}
else if (network == NBitcoin.Altcoins.Litecoin.Instance.Regtest)
{
GolombRiceFilter filter = IndexBuilderService.CreateDummyEmptyFilter(startingHeader.BlockHash);
return(FilterModel.FromLine($"{startingHeader.Height}:{startingHeader.BlockHash}:{filter}:{startingHeader.PrevHash}:{startingHeader.BlockTime.ToUnixTimeSeconds()}"));
}
else
{
throw new NotSupportedNetworkException(network);
}
}
public IndexBuilderService(IRPCClient rpc, BlockNotifier blockNotifier, string indexFilePath)
{
RpcClient = Guard.NotNull(nameof(rpc), rpc);
BlockNotifier = Guard.NotNull(nameof(blockNotifier), blockNotifier);
IndexFilePath = Guard.NotNullOrEmptyOrWhitespace(nameof(indexFilePath), indexFilePath);
Index = new List <FilterModel>();
IndexLock = new AsyncLock();
StartingHeight = SmartHeader.GetStartingHeader(RpcClient.Network).Height;
_serviceStatus = NotStarted;
IoHelpers.EnsureContainingDirectoryExists(IndexFilePath);
// Testing permissions.
using (var _ = File.Open(IndexFilePath, FileMode.OpenOrCreate, FileAccess.ReadWrite))
{
}
if (File.Exists(IndexFilePath))
{
if (RpcClient.Network == Network.RegTest)
{
File.Delete(IndexFilePath); // RegTest is not a global ledger, better to delete it.
}
else
{
foreach (var line in File.ReadAllLines(IndexFilePath))
{
var filter = FilterModel.FromLine(line);
Index.Add(filter);
}
}
}
BlockNotifier.OnBlock += BlockNotifier_OnBlock;
}
public async Task ForeachFiltersAsync(Func <FilterModel, Task> todo, Height fromHeight, CancellationToken cancel = default)
{
using (await MatureIndexFileManager.Mutex.LockAsync(cancel).ConfigureAwait(false))
using (await IndexLock.LockAsync(cancel).ConfigureAwait(false))
{
var firstImmatureHeight = ImmatureFilters.FirstOrDefault()?.Header?.Height;
if (!firstImmatureHeight.HasValue || firstImmatureHeight.Value > fromHeight)
{
if (MatureIndexFileManager.Exists())
{
uint height = StartingHeight;
using var sr = MatureIndexFileManager.OpenText();
if (!sr.EndOfStream)
{
var lineTask = sr.ReadLineAsync();
Task tTask = Task.CompletedTask;
string line = null;
while (lineTask != null)
{
if (firstImmatureHeight == height)
{
break; // Let's use our the immature filters from here on. The content is the same, just someone else modified the file.
}
if (line is null)
{
line = await lineTask.ConfigureAwait(false);
}
lineTask = sr.EndOfStream ? null : sr.ReadLineAsync();
if (height < fromHeight.Value)
{
height++;
line = null;
continue;
}
var filter = FilterModel.FromLine(line);
await tTask.ConfigureAwait(false);
tTask = todo(filter);
height++;
line = null;
}
await tTask.ConfigureAwait(false);
}
while (!sr.EndOfStream)
{
var line = await sr.ReadLineAsync().ConfigureAwait(false);
if (firstImmatureHeight == height)
{
break; // Let's use our the immature filters from here on. The content is the same, just someone else modified the file.
}
if (height < fromHeight.Value)
{
height++;
continue;
}
var filter = FilterModel.FromLine(line);
await todo(filter).ConfigureAwait(false);
height++;
}
}
}
foreach (FilterModel filter in ImmatureFilters)
{
await todo(filter).ConfigureAwait(false);
}
}
}
/// <inheritdoc />
public override object?ReadJson(JsonReader reader, Type objectType, object existingValue, JsonSerializer serializer)
{
var value = Guard.Correct((string)reader.Value);
return(string.IsNullOrWhiteSpace(value) ? default : FilterModel.FromLine(value));
}
public void Synchronize(TimeSpan requestInterval)
{
Guard.NotNull(nameof(requestInterval), requestInterval);
Interlocked.Exchange(ref _running, 1);
Task.Run(async() =>
{
try
{
while (IsRunning)
{
try
{
// If stop was requested return.
if (IsRunning == false)
{
return;
}
FilterModel bestKnownFilter;
using (await IndexLock.LockAsync())
{
bestKnownFilter = Index.Last();
}
var response = await Client.SendAsync(HttpMethod.Get, $"/api/v1/btc/Blockchain/filters/{bestKnownFilter.BlockHash}");
if (response.StatusCode == HttpStatusCode.NoContent)
{
continue;
}
if (response.StatusCode == HttpStatusCode.OK)
{
var filters = await response.Content.ReadAsJsonAsync <List <string> >();
using (await IndexLock.LockAsync())
{
for (int i = 0; i < filters.Count; i++)
{
var filterModel = FilterModel.FromLine(filters[i], bestKnownFilter.BlockHeight + i + 1);
Index.Add(filterModel);
OnNewFilter(filterModel);
}
if (filters.Count == 1) // minor optimization
{
await File.AppendAllLinesAsync(IndexFilePath, new[] { Index.Last().ToLine() });
}
else
{
await File.WriteAllLinesAsync(IndexFilePath, Index.Select(x => x.ToLine()));
}
Logger.LogInfo <IndexDownloader>($"Downloaded filters for blocks from {bestKnownFilter.BlockHeight.Value + 1} to {Index.Last().BlockHeight}.");
}
continue;
}
else if (response.StatusCode == HttpStatusCode.NotFound)
{
// Reorg happened
var reorgedHash = bestKnownFilter.BlockHash;
Logger.LogInfo <IndexDownloader>($"REORG Invalid Block: {reorgedHash}");
// 1. Rollback index
using (await IndexLock.LockAsync())
{
Index.RemoveAt(Index.Count - 1);
}
OnReorg(reorgedHash);
// 2. Serialize Index. (Remove last line.)
var lines = File.ReadAllLines(IndexFilePath);
File.WriteAllLines(IndexFilePath, lines.Take(lines.Length - 1).ToArray());
// 3. Skip the last valid block.
continue;
}
else
{
var error = await response.Content.ReadAsStringAsync();
throw new HttpRequestException($"{response.StatusCode.ToReasonString()}: {error}");
}
}
catch (Exception ex)
{
Logger.LogError <IndexDownloader>(ex);
}
finally
{
await Task.Delay(requestInterval); // Ask for new index in every requestInterval.
}
}
}
finally
{
if (IsStopping)
{
Interlocked.Exchange(ref _running, 3);
}
}
});
}
public async Task FilterBuilderTestAsync()
{
using (var builder = await NodeBuilder.CreateAsync())
{
await builder.CreateNodeAsync();
await builder.StartAllAsync();
CoreNode regtestNode = builder.Nodes[0];
regtestNode.Generate(101);
RPCClient rpc = regtestNode.CreateRpcClient();
var indexBuilderServiceDir = Path.Combine(SharedFixture.DataDir, nameof(IndexBuilderService));
var indexFilePath = Path.Combine(indexBuilderServiceDir, $"Index{rpc.Network}.dat");
var utxoSetFilePath = Path.Combine(indexBuilderServiceDir, $"UtxoSet{rpc.Network}.dat");
var indexBuilderService = new IndexBuilderService(rpc, indexFilePath, utxoSetFilePath);
try
{
indexBuilderService.Synchronize();
// Test initial synchronization.
var times = 0;
uint256 firstHash = await rpc.GetBlockHashAsync(0);
while (indexBuilderService.GetFilterLinesExcluding(firstHash, 102, out _).filters.Count() != 101)
{
if (times > 500) // 30 sec
{
throw new TimeoutException($"{nameof(IndexBuilderService)} test timed out.");
}
await Task.Delay(100);
times++;
}
// Test later synchronization.
regtestNode.Generate(10);
times = 0;
while (indexBuilderService.GetFilterLinesExcluding(firstHash, 112, out bool found5).filters.Count() != 111)
{
Assert.True(found5);
if (times > 500) // 30 sec
{
throw new TimeoutException($"{nameof(IndexBuilderService)} test timed out.");
}
await Task.Delay(100);
times++;
}
// Test correct number of filters is received.
var hundredthHash = await rpc.GetBlockHashAsync(100);
Assert.Equal(11, indexBuilderService.GetFilterLinesExcluding(hundredthHash, 12, out bool found).filters.Count());
Assert.True(found);
var bestHash = await rpc.GetBestBlockHashAsync();
Assert.Empty(indexBuilderService.GetFilterLinesExcluding(bestHash, 1, out bool found2).filters);
Assert.True(found2);
Assert.Empty(indexBuilderService.GetFilterLinesExcluding(uint256.Zero, 1, out bool found3).filters);
Assert.False(found3);
// Test filter block hashes are correct.
var filters = indexBuilderService.GetFilterLinesExcluding(firstHash, 112, out bool found4).filters.ToArray();
Assert.True(found4);
for (int i = 0; i < 111; i++)
{
var expectedHash = await rpc.GetBlockHashAsync(i + 1);
var filterModel = FilterModel.FromLine(filters[i], i);
Assert.Equal(expectedHash, filterModel.BlockHash);
Assert.Null(filterModel.Filter);
}
}
finally
{
if (indexBuilderService != null)
{
await indexBuilderService.StopAsync();
}
}
}
}
public void Synchronize(TimeSpan requestInterval)
{
Guard.NotNull(nameof(requestInterval), requestInterval);
Interlocked.Exchange(ref _running, 1);
Task.Run(async() =>
{
FilterModel bestKnownFilter = null;
try
{
while (IsRunning)
{
try
{
// If stop was requested return.
if (IsRunning == false)
{
return;
}
using (await IndexLock.LockAsync())
{
bestKnownFilter = Index.Last();
}
var filters = await WasabiClient.GetFiltersAsync(bestKnownFilter.BlockHash, 1000);
if (!filters.Any())
{
continue;
}
using (await IndexLock.LockAsync())
{
var filtersList = filters.ToList(); // performance
for (int i = 0; i < filtersList.Count; i++)
{
var filterModel = FilterModel.FromLine(filtersList[i], bestKnownFilter.BlockHeight + i + 1);
Index.Add(filterModel);
NewFilter?.Invoke(this, filterModel);
}
if (filtersList.Count == 1) // minor optimization
{
await File.AppendAllLinesAsync(IndexFilePath, new[] { Index.Last().ToLine() });
}
else
{
await File.WriteAllLinesAsync(IndexFilePath, Index.Select(x => x.ToLine()));
}
Logger.LogInfo <IndexDownloader>($"Downloaded filters for blocks from {bestKnownFilter.BlockHeight.Value + 1} to {Index.Last().BlockHeight}.");
}
continue;
}
catch (HttpRequestException ex) when(ex.Message.StartsWith(HttpStatusCode.NotFound.ToReasonString()))
{
// Reorg happened
var reorgedHash = bestKnownFilter.BlockHash;
Logger.LogInfo <IndexDownloader>($"REORG Invalid Block: {reorgedHash}");
// 1. Rollback index
using (await IndexLock.LockAsync())
{
Index.RemoveAt(Index.Count - 1);
}
Reorged?.Invoke(this, reorgedHash);
// 2. Serialize Index. (Remove last line.)
var lines = File.ReadAllLines(IndexFilePath);
File.WriteAllLines(IndexFilePath, lines.Take(lines.Length - 1).ToArray());
// 3. Skip the last valid block.
continue;
}
catch (Exception ex)
{
Logger.LogError <IndexDownloader>(ex);
}
finally
{
await Task.Delay(requestInterval); // Ask for new index in every requestInterval.
}
}
}
finally
{
if (IsStopping)
{
Interlocked.Exchange(ref _running, 3);
}
}
});
}