public async Task InitializeAsync(CancellationToken cancel = default)
{
using (BenchmarkLogger.Measure())
{
using (await IndexLock.LockAsync().ConfigureAwait(false))
using (await MatureIndexAsyncLock.LockAsync().ConfigureAwait(false))
using (await ImmatureIndexAsyncLock.LockAsync().ConfigureAwait(false))
{
IoHelpers.EnsureDirectoryExists(WorkFolderPath);
cancel.ThrowIfCancellationRequested();
await EnsureBackwardsCompatibilityAsync().ConfigureAwait(false);
if (Network == Network.RegTest)
{
MatureIndexFileManager.DeleteMe(); // RegTest is not a global ledger, better to delete it.
ImmatureIndexFileManager.DeleteMe();
}
cancel.ThrowIfCancellationRequested();
if (!MatureIndexFileManager.Exists())
{
await MatureIndexFileManager.WriteAllLinesAsync(new[] { StartingFilter.ToLine() }).ConfigureAwait(false);
}
cancel.ThrowIfCancellationRequested();
await InitializeFiltersAsync(cancel).ConfigureAwait(false);
}
}
}
private async Task InitializeFiltersAsync()
{
try
{
if (MatureIndexFileManager.Exists())
{
using var sr = MatureIndexFileManager.OpenText();
if (!sr.EndOfStream)
{
var lineTask = sr.ReadLineAsync();
string line = null;
while (lineTask != null)
{
if (line is null)
{
line = await lineTask.ConfigureAwait(false);
}
lineTask = sr.EndOfStream ? null : sr.ReadLineAsync();
ProcessLine(line, enqueue: false);
line = null;
}
}
}
}
catch
{
// We found a corrupted entry. Stop here.
// Delete the currupted file.
// Do not try to autocorrect, because the internal data structures are throwing events that may confuse the consumers of those events.
Logger.LogError("Mature index got corrupted. Deleting both mature and immature index...");
MatureIndexFileManager.DeleteMe();
ImmatureIndexFileManager.DeleteMe();
throw;
}
try
{
if (ImmatureIndexFileManager.Exists())
{
foreach (var line in await ImmatureIndexFileManager.ReadAllLinesAsync().ConfigureAwait(false)) // We can load ImmatureIndexFileManager to the memory, no problem.
{
ProcessLine(line, enqueue: true);
}
}
}
catch
{
// We found a corrupted entry. Stop here.
// Delete the currupted file.
// Do not try to autocorrect, because the internal data structures are throwing events that may confuse the consumers of those events.
Logger.LogError("Immature index got corrupted. Deleting it...");
ImmatureIndexFileManager.DeleteMe();
throw;
}
}
/// <summary>
/// It'll LogError the exceptions.
/// If cancelled, it'll LogTrace the exception.
/// </summary>
private async Task TryCommitToFileAsync(TimeSpan throttle, CancellationToken cancel)
{
try
{
// If throttle is requested, then throttle.
if (throttle != TimeSpan.Zero)
{
// Increment the throttle ID and remember the incremented value.
int incremented = Interlocked.Increment(ref _throttleId);
if (incremented < 21)
{
await Task.Delay(throttle, cancel);
}
// If the _throttleId is still the incremented value, then I am the latest CommitToFileAsync request.
// In this case I want to make the _throttledId 0 and go ahead and do the writeline.
// If the _throttledId is not the incremented value anymore then I am not the latest request here,
// So just return, the latest request will do the file write in its own time.
if (Interlocked.CompareExchange(ref _throttleId, 0, incremented) != incremented)
{
return;
}
}
else
{
Interlocked.Exchange(ref _throttleId, 0); // So to notified the currently throttled threads that they don't have to run.
}
using (await MatureIndexFileManager.Mutex.LockAsync(cancel))
using (await ImmatureIndexFileManager.Mutex.LockAsync(cancel))
using (await IndexLock.LockAsync(cancel))
{
// Don't feed the cancellationToken here I always want this to finish running for safety.
var currentImmatureLines = ImmatureFilters.Select(x => x.ToHeightlessLine());
var matureLinesToAppend = currentImmatureLines.SkipLast(100);
var immatureLines = currentImmatureLines.TakeLast(100);
await MatureIndexFileManager.AppendAllLinesAsync(matureLinesToAppend);
await ImmatureIndexFileManager.WriteAllLinesAsync(immatureLines);
while (ImmatureFilters.Count > 100)
{
ImmatureFilters.RemoveFirst();
}
}
}
catch (Exception ex) when(ex is OperationCanceledException ||
ex is TaskCanceledException ||
ex is TimeoutException)
{
Logger.LogTrace <IndexStore>(ex);
}
catch (Exception ex)
{
Logger.LogError <IndexStore>(ex);
}
}
private async Task EnsureBackwardsCompatibilityAsync()
{
try
{
// Before Wasabi 1.1.5
var oldIndexFilePath = Path.Combine(EnvironmentHelpers.GetDataDir(Path.Combine("WalletWasabi", "Client")), $"Index{Network}.dat");
// Before Wasabi 1.1.6
var oldFileNames = new[]
{
"ImmatureIndex.dat",
"ImmatureIndex.dat.dig",
"MatureIndex.dat",
"MatureIndex.dat.dig"
};
var oldIndexFolderPath = Path.Combine(EnvironmentHelpers.GetDataDir(Path.Combine("WalletWasabi", "Client")), "BitcoinStore", Network.ToString());
foreach (var fileName in oldFileNames)
{
var oldFilePath = Path.Combine(oldIndexFolderPath, fileName);
if (File.Exists(oldFilePath))
{
string newFilePath = oldFilePath.Replace(oldIndexFolderPath, WorkFolderPath);
if (File.Exists(newFilePath))
{
File.Delete(newFilePath);
}
File.Move(oldFilePath, newFilePath);
}
}
if (File.Exists(oldIndexFilePath))
{
string[] allLines = await File.ReadAllLinesAsync(oldIndexFilePath).ConfigureAwait(false);
var matureLines = allLines.SkipLast(100);
var immatureLines = allLines.TakeLast(100);
await MatureIndexFileManager.WriteAllLinesAsync(matureLines).ConfigureAwait(false);
await ImmatureIndexFileManager.WriteAllLinesAsync(immatureLines).ConfigureAwait(false);
File.Delete(oldIndexFilePath);
}
await DeleteIfDeprecatedAsync().ConfigureAwait(false);
}
catch (Exception ex)
{
Logger.LogWarning($"Backwards compatibility could not be ensured. Exception: {ex}.");
}
}
private async Task DeleteIfDeprecatedAsync()
{
if (MatureIndexFileManager.Exists())
{
await DeleteIfDeprecatedAsync(MatureIndexFileManager).ConfigureAwait(false);
}
if (ImmatureIndexFileManager.Exists())
{
await DeleteIfDeprecatedAsync(ImmatureIndexFileManager).ConfigureAwait(false);
}
}
private async Task InitializeFiltersAsync()
{
try
{
Height height = StartingHeight;
if (MatureIndexFileManager.Exists())
{
using (var sr = MatureIndexFileManager.OpenText())
{
if (!sr.EndOfStream)
{
var lineTask = sr.ReadLineAsync();
string line = null;
while (lineTask != null)
{
if (line is null)
{
line = await lineTask;
}
lineTask = sr.EndOfStream ? null : sr.ReadLineAsync();
ProcessLine(height, line, enqueue: false);
height++;
line = null;
}
}
}
}
if (ImmatureIndexFileManager.Exists())
{
foreach (var line in await ImmatureIndexFileManager.ReadAllLinesAsync()) // We can load ImmatureIndexFileManager to the memory, no problem.
{
ProcessLine(height, line, enqueue: true);
height++;
}
}
}
catch
{
// We found a corrupted entry. Stop here.
// Delete the currupted file.
// Don't try to autocorrect, because the internal data structures are throwing events those may confuse the consumers of those events.
Logger.LogError <IndexStore>("An index file got corrupted. Deleting index files...");
MatureIndexFileManager.DeleteMe();
ImmatureIndexFileManager.DeleteMe();
throw;
}
}
private async Task DeleteIfDeprecatedAsync(DigestableSafeIoManager ioManager)
{
string?firstLine;
using (var content = ioManager.OpenText())
{
firstLine = await content.ReadLineAsync().ConfigureAwait(false);
}
try
{
FilterModel.FromLine(firstLine);
}
catch
{
Logger.LogWarning("Old Index file detected. Deleting it.");
MatureIndexFileManager.DeleteMe();
ImmatureIndexFileManager.DeleteMe();
Logger.LogWarning("Successfully deleted old Index file.");
}
}
public async Task <IEnumerable <FilterModel> > RemoveAllImmmatureFiltersAsync(CancellationToken cancel, bool deleteAndCrashIfMature = false)
{
var removed = new List <FilterModel>();
using (await IndexLock.LockAsync(cancel).ConfigureAwait(false))
{
if (ImmatureFilters.Any())
{
Logger.LogWarning($"Filters got corrupted. Reorging {ImmatureFilters.Count} immature filters in an attempt to fix them.");
}
else
{
Logger.LogCritical($"Filters got corrupted and have no more immature filters.");
if (deleteAndCrashIfMature)
{
Logger.LogCritical($"Deleting all filters and crashing the software...");
using (await MatureIndexAsyncLock.LockAsync(cancel).ConfigureAwait(false))
using (await ImmatureIndexAsyncLock.LockAsync(cancel).ConfigureAwait(false))
{
ImmatureIndexFileManager.DeleteMe();
MatureIndexFileManager.DeleteMe();
}
Environment.Exit(2);
}
}
}
while (ImmatureFilters.Any())
{
removed.Add(await RemoveLastFilterAsync(cancel).ConfigureAwait(false));
}
return(removed);
}
private async Task InitializeFiltersAsync(CancellationToken cancel)
{
try
{
if (MatureIndexFileManager.Exists())
{
using (BenchmarkLogger.Measure(LogLevel.Debug, "MatureIndexFileManager loading"))
{
int i = 0;
using StreamReader sr = MatureIndexFileManager.OpenText();
if (!sr.EndOfStream)
{
while (true)
{
i++;
cancel.ThrowIfCancellationRequested();
string?line = await sr.ReadLineAsync().ConfigureAwait(false);
if (line is null)
{
break;
}
ProcessLine(line, enqueue: false);
}
}
Logger.LogDebug($"Loaded {i} lines from the mature index file.");
}
}
}
catch (Exception ex) when(ex is not OperationCanceledException)
{
// We found a corrupted entry. Stop here.
// Delete the corrupted file.
// Do not try to autocorrect, because the internal data structures are throwing events that may confuse the consumers of those events.
Logger.LogError("Mature index got corrupted. Deleting both mature and immature index...");
MatureIndexFileManager.DeleteMe();
ImmatureIndexFileManager.DeleteMe();
throw;
}
cancel.ThrowIfCancellationRequested();
try
{
if (ImmatureIndexFileManager.Exists())
{
foreach (var line in await ImmatureIndexFileManager.ReadAllLinesAsync(cancel).ConfigureAwait(false)) // We can load ImmatureIndexFileManager to the memory, no problem.
{
ProcessLine(line, enqueue: true);
cancel.ThrowIfCancellationRequested();
}
}
}
catch (Exception ex) when(ex is not OperationCanceledException)
{
// We found a corrupted entry. Stop here.
// Delete the corrupted file.
// Do not try to autocorrect, because the internal data structures are throwing events that may confuse the consumers of those events.
Logger.LogError("Immature index got corrupted. Deleting it...");
ImmatureIndexFileManager.DeleteMe();
throw;
}
}
