public void Initialize()
{
// The transaction index is needed in the event of a reorg.
if (!this.storeSettings.AddressIndex)
{
this.logger.LogTrace("(-)[DISABLED]");
return;
}
string dbPath = Path.Combine(this.dataFolder.RootPath, AddressIndexerDatabaseFilename);
FileMode fileMode = RuntimeInformation.IsOSPlatform(OSPlatform.OSX) ? FileMode.Exclusive : FileMode.Shared;
this.db = new LiteDatabase(new ConnectionString()
{
Filename = dbPath, Mode = fileMode
});
this.addressIndexRepository = new AddressIndexRepository(this.db, this.loggerFactory);
this.logger.LogDebug("Address indexing is enabled.");
this.tipDataStore = this.db.GetCollection <AddressIndexerTipData>(DbTipDataKey);
lock (this.lockObject)
{
AddressIndexerTipData tipData = this.tipDataStore.FindAll().FirstOrDefault();
this.logger.LogDebug("Tip data: '{0}'.", tipData == null ? "null" : tipData.ToString());
this.IndexerTip = tipData == null ? this.chainIndexer.Genesis : this.consensusManager.Tip.FindAncestorOrSelf(new uint256(tipData.TipHashBytes));
if (this.IndexerTip == null)
{
// This can happen if block hash from tip data is no longer a part of the consensus chain and node was killed in the middle of a reorg.
int rewindAmount = this.compactionTriggerDistance / 2;
if (rewindAmount > this.consensusManager.Tip.Height)
{
this.IndexerTip = this.chainIndexer.Genesis;
}
else
{
this.IndexerTip = this.consensusManager.Tip.GetAncestor(this.consensusManager.Tip.Height - rewindAmount);
}
}
}
this.outpointsRepository = new AddressIndexerOutpointsRepository(this.db, this.loggerFactory);
this.RewindAndSave(this.IndexerTip);
this.logger.LogDebug("Indexer initialized at '{0}'.", this.IndexerTip);
this.indexingTask = Task.Run(async() => await this.IndexAddressesContinuouslyAsync().ConfigureAwait(false));
this.asyncProvider.RegisterTask($"{nameof(AddressIndexer)}.{nameof(this.indexingTask)}", this.indexingTask);
this.nodeStats.RegisterStats(this.AddInlineStats, StatsType.Inline, this.GetType().Name, 400);
}
public void OutPointCacheCanRetrieveExisting()
{
const string CollectionName = "DummyCollection";
var dataFolder = new DataFolder(TestBase.CreateTestDir(this));
string dbPath = Path.Combine(dataFolder.RootPath, CollectionName);
FileMode fileMode = RuntimeInformation.IsOSPlatform(OSPlatform.OSX) ? FileMode.Exclusive : FileMode.Shared;
var database = new LiteDatabase(new ConnectionString()
{
Filename = dbPath, Mode = fileMode
});
var cache = new AddressIndexerOutpointsRepository(database, new ExtendedLoggerFactory());
var outPoint = new OutPoint(uint256.Parse("0000af9ab2c8660481328d0444cf167dfd31f24ca2dbba8e5e963a2434cffa93"), 0);
var data = new OutPointData()
{
Outpoint = outPoint.ToString(), ScriptPubKeyBytes = new byte[] { 0, 0, 0, 0 }, Money = Money.Coins(1)
};
cache.AddOutPointData(data);
Assert.True(cache.TryGetOutPointData(outPoint, out OutPointData retrieved));
Assert.NotNull(retrieved);
Assert.Equal(outPoint.ToString(), retrieved.Outpoint);
}
private LiteDatabase Connect(LiteDB.FileMode fileMode)
=> _mem != null
? new LiteDatabase(_mem)
: new LiteDatabase(new ConnectionString
{
Filename = DbPath,
Journal = false,
Mode = fileMode,
LimitSize = long.MaxValue
});
public AddressIndexerOutpointsRepositoryTests()
{
LiteDB.FileMode fileMode = RuntimeInformation.IsOSPlatform(OSPlatform.OSX) ? LiteDB.FileMode.Exclusive : LiteDB.FileMode.Shared;
var db = new LiteDatabase(new ConnectionString()
{
Filename = this.RandomString(20) + ".db", Upgrade = true, Mode = fileMode
});
this.repository = new AddressIndexerOutpointsRepository(db, new ExtendedLoggerFactory(), this.maxItems);
}
public void Initialize()
{
// The transaction index is needed in the event of a reorg.
if (!this.storeSettings.AddressIndex)
{
this.logger.LogTrace("(-)[DISABLED]");
return;
}
string dbPath = Path.Combine(this.dataFolder.RootPath, AddressIndexerDatabaseFilename);
FileMode fileMode = RuntimeInformation.IsOSPlatform(OSPlatform.OSX) ? FileMode.Exclusive : FileMode.Shared;
this.db = new LiteDatabase(new ConnectionString()
{
Filename = dbPath, Mode = fileMode
});
this.addressIndexRepository = new AddressIndexRepository(this.db, this.loggerFactory);
this.logger.LogDebug("Address indexing is enabled.");
this.tipDataStore = this.db.GetCollection <AddressIndexerTipData>(DbTipDataKey);
lock (this.lockObject)
{
this.tipData = this.tipDataStore.FindAll().FirstOrDefault();
if (this.tipData == null)
{
this.logger.LogDebug("Tip was not found, initializing with genesis.");
this.tipData = new AddressIndexerTipData()
{
TipHashBytes = this.network.GenesisHash.ToBytes(), Height = 0
};
this.tipDataStore.Insert(this.tipData);
}
this.IndexerTip = this.consensusManager.Tip.FindAncestorOrSelf(new uint256(this.tipData.TipHashBytes));
}
this.outpointsRepository = new AddressIndexerOutpointsRepository(this.db, this.loggerFactory);
if (this.IndexerTip == null)
{
this.IndexerTip = this.consensusManager.Tip.GetAncestor(0);
}
this.indexingTask = Task.Run(async() => await this.IndexAddressesContinuouslyAsync().ConfigureAwait(false));
this.asyncProvider.RegisterTask($"{nameof(AddressIndexer)}.{nameof(this.indexingTask)}", this.indexingTask);
this.nodeStats.RegisterStats(this.AddInlineStats, StatsType.Inline, 400);
}
public WalletStore(Network network, DataFolder dataFolder, Types.Wallet wallet)
{
var dbPath = Path.Combine(dataFolder.WalletFolderPath, $"{wallet.Name}.db");
if (!Directory.Exists(dataFolder.WalletFolderPath))
{
Directory.CreateDirectory(dataFolder.WalletFolderPath);
}
BsonMapper mapper = this.Create();
LiteDB.FileMode fileMode = RuntimeInformation.IsOSPlatform(OSPlatform.OSX) ? LiteDB.FileMode.Exclusive : LiteDB.FileMode.Shared;
if (!File.Exists(dbPath))
{
this.db = new LiteDatabase(new ConnectionString()
{
Filename = dbPath, Mode = fileMode
}, mapper: mapper);
}
else
{
// Only perform this check if the database file already exists.
this.db = new LiteDatabase(new ConnectionString()
{
Filename = dbPath, Mode = fileMode
}, mapper: mapper);
// Attempt to access the user version, this will crash if the loaded database is V5 and we use V4 packages.
try
{
var userVersion = this.db.Engine.UserVersion;
}
catch (LiteDB.LiteException)
{
var dbBackupPath = Path.Combine(dataFolder.WalletFolderPath, $"{wallet.Name}.error.db");
// Move the problematic database file, which might be a V5 database.
File.Move(dbPath, dbBackupPath);
// Re-create the database object after we renamed the file.
this.db = new LiteDatabase(new ConnectionString()
{
Filename = dbPath, Mode = fileMode
}, mapper: mapper);
}
}
this.network = network;
this.Init(wallet);
}
public void Initialize()
{
if (!this.storeSettings.TxIndex || !this.storeSettings.AddressIndex)
{
this.logger.LogTrace("(-)[DISABLED]");
return;
}
string dbPath = Path.Combine(this.dataFolder.RootPath, "addressindex.litedb");
FileMode fileMode = RuntimeInformation.IsOSPlatform(OSPlatform.OSX) ? FileMode.Exclusive : FileMode.Shared;
this.db = new LiteDatabase(new ConnectionString()
{
Filename = dbPath, Mode = fileMode
});
this.logger.LogDebug("TxIndexing is enabled.");
this.dataStore = this.db.GetCollection <AddressIndexerData>(DbKey);
lock (this.lockObject)
{
this.addressesIndex = this.dataStore.FindAll().FirstOrDefault();
if (this.addressesIndex == null)
{
this.logger.LogDebug("Tip was not found, initializing with genesis.");
this.addressesIndex = new AddressIndexerData()
{
TipHashBytes = this.network.GenesisHash.ToBytes(),
AddressChanges = new Dictionary <string, List <AddressBalanceChange> >()
};
this.dataStore.Insert(this.addressesIndex);
}
this.IndexerTip = this.consensusManager.Tip.FindAncestorOrSelf(new uint256(this.addressesIndex.TipHashBytes));
}
if (this.IndexerTip == null)
{
this.IndexerTip = this.consensusManager.Tip.GetAncestor(0);
}
this.indexingTask = Task.Run(async() => await this.IndexAddressesContinuouslyAsync().ConfigureAwait(false));
this.nodeStats.RegisterStats(this.AddInlineStats, StatsType.Inline, 400);
}
public void OutPointCacheCannotRetrieveNonexistent()
{
const string CollectionName = "DummyCollection";
var dataFolder = new DataFolder(TestBase.CreateTestDir(this));
string dbPath = Path.Combine(dataFolder.RootPath, CollectionName);
FileMode fileMode = RuntimeInformation.IsOSPlatform(OSPlatform.OSX) ? FileMode.Exclusive : FileMode.Shared;
var database = new LiteDatabase(new ConnectionString()
{
Filename = dbPath, Mode = fileMode
});
var cache = new AddressIndexerOutpointsRepository(database, new ExtendedLoggerFactory());
Assert.False(cache.TryGetOutPointData(new OutPoint(uint256.Parse("0000af9ab2c8660481328d0444cf167dfd31f24ca2dbba8e5e963a2434cffa93"), 1), out OutPointData retrieved));
Assert.Null(retrieved);
}
private static LiteDatabase Connect <T>(this LiteRevisionsDB <T> revDb, LiteDB.FileMode fileMode)
{
var db = revDb._mem != null
? new LiteDatabase(revDb._mem)
: new LiteDatabase(new ConnectionString
{
Filename = revDb.DbPath,
Journal = false,
Mode = fileMode,
LimitSize = long.MaxValue
});
if (fileMode != LiteDB.FileMode.ReadOnly)
{
db.Version1 <T>().EnsureIndex(nameof(Versioned <T> .GroupId));
}
return(db);
}
public void AddressCacheRetrievesBlankRecordForNonexistent()
{
const string CollectionName = "DummyCollection";
var dataFolder = new DataFolder(TestBase.CreateTestDir(this));
string dbPath = Path.Combine(dataFolder.RootPath, CollectionName);
FileMode fileMode = RuntimeInformation.IsOSPlatform(OSPlatform.OSX) ? FileMode.Exclusive : FileMode.Shared;
var database = new LiteDatabase(new ConnectionString()
{
Filename = dbPath, Mode = fileMode
});
var cache = new AddressIndexRepository(database, new ExtendedLoggerFactory());
AddressIndexerData retrieved = cache.GetOrCreateAddress("xyz");
// A record will be returned with no balance changes associated, if it is new.
Assert.NotNull(retrieved);
Assert.Equal("xyz", retrieved.Address);
Assert.Empty(retrieved.BalanceChanges);
}
/// <inheritdoc />
public void Initialize()
{
if (this.db != null)
{
throw new Exception("NFTTransferIndexer already initialized!");
}
string dbPath = Path.Combine(this.dataFolder.RootPath, DatabaseFilename);
FileMode fileMode = RuntimeInformation.IsOSPlatform(OSPlatform.OSX) ? FileMode.Exclusive : FileMode.Shared;
this.db = new LiteDatabase(new ConnectionString()
{
Filename = dbPath, Mode = fileMode
});
this.NFTContractCollection = this.db.GetCollection <NFTContractModel>(DbOwnedNFTsKey);
this.indexingTask = Task.Run(async() => await this.IndexNFTsContinuouslyAsync().ConfigureAwait(false));
this.asyncProvider.RegisterTask($"{nameof(AddressIndexer)}.{nameof(this.indexingTask)}", this.indexingTask);
}
public void AddressCacheCanRetrieveExisting()
{
const string CollectionName = "DummyCollection";
var dataFolder = new DataFolder(TestBase.CreateTestDir(this));
string dbPath = Path.Combine(dataFolder.RootPath, CollectionName);
FileMode fileMode = RuntimeInformation.IsOSPlatform(OSPlatform.OSX) ? FileMode.Exclusive : FileMode.Shared;
var database = new LiteDatabase(new ConnectionString()
{
Filename = dbPath, Mode = fileMode
});
var cache = new AddressIndexRepository(database, new ExtendedLoggerFactory());
string address = "xyz";
var balanceChanges = new List <AddressBalanceChange>();
balanceChanges.Add(new AddressBalanceChange()
{
BalanceChangedHeight = 1, Deposited = true, Satoshi = 1
});
var data = new AddressIndexerData()
{
Address = address, BalanceChanges = balanceChanges
};
cache.AddOrUpdate(data.Address, data, data.BalanceChanges.Count + 1);
AddressIndexerData retrieved = cache.GetOrCreateAddress("xyz");
Assert.NotNull(retrieved);
Assert.Equal("xyz", retrieved.Address);
Assert.Equal(1, retrieved.BalanceChanges.First().BalanceChangedHeight);
Assert.True(retrieved.BalanceChanges.First().Deposited);
Assert.Equal(1, retrieved.BalanceChanges.First().Satoshi);
}
public void AddressCacheEvicts()
{
const string CollectionName = "AddrData";
var dataFolder = new DataFolder(TestBase.CreateTestDir(this));
string dbPath = Path.Combine(dataFolder.RootPath, CollectionName);
FileMode fileMode = RuntimeInformation.IsOSPlatform(OSPlatform.OSX) ? FileMode.Exclusive : FileMode.Shared;
var database = new LiteDatabase(new ConnectionString()
{
Filename = dbPath, Mode = fileMode
});
var cache = new AddressIndexRepository(database, new ExtendedLoggerFactory(), 4);
// Recall, each index entry counts as 1 and each balance change associated with it is an additional 1.
Assert.Equal(0, database.GetCollection <AddressIndexerData>(CollectionName).Count());
string address1 = "xyz";
var balanceChanges1 = new List <AddressBalanceChange>();
balanceChanges1.Add(new AddressBalanceChange()
{
BalanceChangedHeight = 1, Deposited = true, Satoshi = 1
});
var data1 = new AddressIndexerData()
{
Address = address1, BalanceChanges = balanceChanges1
};
cache.AddOrUpdate(data1.Address, data1, data1.BalanceChanges.Count + 1);
Assert.Equal(0, database.GetCollection <AddressIndexerData>(CollectionName).Count());
string address2 = "abc";
var balanceChanges2 = new List <AddressBalanceChange>();
balanceChanges2.Add(new AddressBalanceChange()
{
BalanceChangedHeight = 2, Deposited = false, Satoshi = 2
});
cache.AddOrUpdate(address2, new AddressIndexerData()
{
Address = address2, BalanceChanges = balanceChanges2
}, balanceChanges2.Count + 1);
Assert.Equal(0, database.GetCollection <AddressIndexerData>(CollectionName).Count());
string address3 = "def";
var balanceChanges3 = new List <AddressBalanceChange>();
balanceChanges3.Add(new AddressBalanceChange()
{
BalanceChangedHeight = 3, Deposited = true, Satoshi = 3
});
cache.AddOrUpdate(address3, new AddressIndexerData()
{
Address = address3, BalanceChanges = balanceChanges3
}, balanceChanges3.Count + 1);
// One of the cache items should have been evicted, and will therefore be persisted on disk.
Assert.Equal(1, database.GetCollection <AddressIndexerData>(CollectionName).Count());
// The evicted item should be data1.
Assert.Equal(data1.Address, database.GetCollection <AddressIndexerData>(CollectionName).FindAll().First().Address);
Assert.Equal(1, database.GetCollection <AddressIndexerData>(CollectionName).FindAll().First().BalanceChanges.First().BalanceChangedHeight);
Assert.True(database.GetCollection <AddressIndexerData>(CollectionName).FindAll().First().BalanceChanges.First().Deposited);
Assert.Equal(1, database.GetCollection <AddressIndexerData>(CollectionName).FindAll().First().BalanceChanges.First().Satoshi);
// Check that the first address can still be retrieved, it should come from disk in this case.
AddressIndexerData retrieved = cache.GetOrCreateAddress("xyz");
Assert.NotNull(retrieved);
Assert.Equal("xyz", retrieved.Address);
Assert.Equal(1, retrieved.BalanceChanges.First().BalanceChangedHeight);
Assert.True(retrieved.BalanceChanges.First().Deposited);
Assert.Equal(1, retrieved.BalanceChanges.First().Satoshi);
}
public void OutPointCacheEvicts()
{
const string CollectionName = "OutputsData";
var dataFolder = new DataFolder(TestBase.CreateTestDir(this));
string dbPath = Path.Combine(dataFolder.RootPath, CollectionName);
FileMode fileMode = RuntimeInformation.IsOSPlatform(OSPlatform.OSX) ? FileMode.Exclusive : FileMode.Shared;
var database = new LiteDatabase(new ConnectionString()
{
Filename = dbPath, Mode = fileMode
});
var cache = new AddressIndexerOutpointsRepository(database, new ExtendedLoggerFactory(), 2);
Assert.Equal(0, cache.Count);
Assert.Equal(0, database.GetCollection <OutPointData>(CollectionName).Count());
var outPoint1 = new OutPoint(uint256.Parse("0000af9ab2c8660481328d0444cf167dfd31f24ca2dbba8e5e963a2434cffa93"), 1);;
var pair1 = new OutPointData()
{
Outpoint = outPoint1.ToString(), ScriptPubKeyBytes = new byte[] { 0, 0, 0, 0 }, Money = Money.Coins(1)
};
cache.AddOutPointData(pair1);
Assert.Equal(1, cache.Count);
Assert.Equal(0, database.GetCollection <OutPointData>(CollectionName).Count());
var outPoint2 = new OutPoint(uint256.Parse("cf8ce1419bbc4870b7d4f1c084534d91126dd3283b51ec379e0a20e27bd23633"), 2);;
var pair2 = new OutPointData()
{
Outpoint = outPoint2.ToString(), ScriptPubKeyBytes = new byte[] { 1, 1, 1, 1 }, Money = Money.Coins(2)
};
cache.AddOutPointData(pair2);
Assert.Equal(2, cache.Count);
Assert.Equal(0, database.GetCollection <OutPointData>(CollectionName).Count());
var outPoint3 = new OutPoint(uint256.Parse("126dd3283b51ec379e0a20e27bd23633cf8ce1419bbc4870b7d4f1c084534d91"), 3);;
var pair3 = new OutPointData()
{
Outpoint = outPoint3.ToString(), ScriptPubKeyBytes = new byte[] { 2, 2, 2, 2 }, Money = Money.Coins(3)
};
cache.AddOutPointData(pair3);
Assert.Equal(2, cache.Count);
// One of the cache items should have been evicted, and will therefore be persisted on disk.
Assert.Equal(1, database.GetCollection <OutPointData>(CollectionName).Count());
// The evicted item should be pair1.
Assert.Equal(pair1.ScriptPubKeyBytes, database.GetCollection <OutPointData>(CollectionName).FindAll().First().ScriptPubKeyBytes);
// It should still be possible to retrieve pair1 from the cache (it will pull it from disk).
Assert.True(cache.TryGetOutPointData(outPoint1, out OutPointData pair1AfterEviction));
Assert.NotNull(pair1AfterEviction);
Assert.Equal(pair1.ScriptPubKeyBytes, pair1AfterEviction.ScriptPubKeyBytes);
Assert.Equal(pair1.Money, pair1AfterEviction.Money);
}
