public IndexerTester(string folder)
{
TestUtils.EnsureNew(folder);
var config = AzureIndexerLoop.IndexerConfigFromSettings(
new AzureIndexerSettings()
{
StorageNamespace = folder
}, Network.TestNet, new LoggerFactory());
config.EnsureSetup();
_Importer = config.CreateIndexer();
List <Task> creating = new List <Task>();
foreach (var table in config.EnumerateTables())
{
creating.Add(table.CreateIfNotExistsAsync());
}
creating.Add(config.GetBlocksContainer().CreateIfNotExistsAsync());
Task.WaitAll(creating.ToArray());
_Folder = folder;
}
static void Main(string[] args)
{
try
{
var options = new IndexerOptions();
if (args.Length == 0)
{
System.Console.WriteLine(options.GetUsage());
}
if (Parser.Default.ParseArguments(args, options))
{
System.Console.WriteLine("NBitcoin.Indexer " + typeof(AzureIndexer).Assembly.GetName().Version);
var indexer = AzureIndexer.CreateIndexer();
indexer.CheckpointInterval = TimeSpan.Parse(options.CheckpointInterval);
indexer.NoSave = options.NoSave;
indexer.FromHeight = options.From;
indexer.ToHeight = options.To;
indexer.TaskCount = options.ThreadCount;
ChainBase chain = null;
if (options.IndexBlocks)
{
chain = chain ?? indexer.GetNodeChain();
indexer.IndexBlocks(chain);
}
if (options.IndexChain)
{
chain = chain ?? indexer.GetNodeChain();
indexer.IndexChain(chain);
}
if (options.IndexTransactions)
{
chain = chain ?? indexer.GetNodeChain();
indexer.IndexTransactions(chain);
}
if (options.IndexAddresses)
{
chain = chain ?? indexer.GetNodeChain();
indexer.IndexOrderedBalances(chain);
}
if (options.IndexWallets)
{
chain = chain ?? indexer.GetNodeChain();
indexer.IndexWalletBalances(chain);
}
}
}
catch (ConfigurationErrorsException ex)
{
System.Console.WriteLine("LocalSettings.config missing settings : " + ex.Message);
}
}
public IndexerTester(string folder)
{
TestUtils.EnsureNew(folder);
var config = IndexerConfiguration.FromConfiguration();
config.Network = Network.TestNet;
config.StorageNamespace = folder;
_Importer = config.CreateIndexer();
foreach (var table in config.EnumerateTables())
{
table.CreateIfNotExists();
}
config.GetBlocksContainer().CreateIfNotExists();
config.EnsureSetup();
_Folder = folder;
}
public async Task MapsSourceToSearchDocument()
{
var index = new Index(); //for proper use, this index should have been prepopulated
var mockSearchIndexClient = new SearchIndexClientMock <SearchDocument>();
var mappedSearchDocument = new SearchDocument();
var indexer = new AzureIndexer <Source, SearchDocument>(
mockSearchIndexClient.SearchIndexClient, (tfr) => mappedSearchDocument);
var source = new Source();
await indexer.IndexAsync(source);
Assert.Single(mockSearchIndexClient.IndexedBatches);
var firstIndexAction = mockSearchIndexClient.IndexedBatches[0].Actions.First();
Assert.Same(mappedSearchDocument, firstIndexAction.Document);
}
public IndexerTester(string folder)
{
TestUtils.EnsureNew(folder);
var config = IndexerConfiguration.FromConfiguration();
config.StorageNamespace = folder;
config.MainDirectory = folder;
_Importer = config.CreateIndexer();
foreach (var table in config.EnumerateTables())
{
table.CreateIfNotExists();
}
config.GetBlocksContainer().CreateIfNotExists();
config.EnsureSetup();
_Folder = folder;
}
public IndexerTester(string folder)
{
TestUtils.EnsureNew(folder);
var config = CreateConfiguration();
config.Network = Network.TestNet;
config.StorageNamespace = folder;
_Importer = config.CreateIndexer();
List<Task> creating = new List<Task>();
foreach (var table in config.EnumerateTables())
{
creating.Add(table.CreateIfNotExistsAsync());
}
creating.Add(config.GetBlocksContainer().CreateIfNotExistsAsync());
Task.WaitAll(creating.ToArray());
config.EnsureSetup();
_Folder = folder;
}
public void Listen(ConcurrentChain chain = null)
{
ListenerTrace.Info($"Connecting to node {_Configuration.Indexer.Node}");
var ip = Utils.ParseIpEndpoint(_Configuration.Indexer.Node, Configuration.Indexer.Network.DefaultPort);
ListenerTrace.Info($"Connecting to node ip {ip.ToString()}");
var node = Node.Connect(Configuration.Indexer.Network, ip);
ListenerTrace.Info($"Connected, trying handshake...");
node.VersionHandshake();
ListenerTrace.Info($"Hanshaked");
node.Disconnect();
_Chain = new ConcurrentChain(_Configuration.Indexer.Network);
_Indexer = Configuration.Indexer.CreateIndexer();
if (chain == null)
{
chain = new ConcurrentChain(_Configuration.Indexer.Network);
}
_Chain = chain;
ListenerTrace.Info("Fetching headers from " + _Chain.Tip.Height + " (from azure)");
var client = Configuration.Indexer.CreateIndexerClient();
client.SynchronizeChain(chain);
ListenerTrace.Info("Headers fetched tip " + _Chain.Tip.Height);
_Disposables.Add(_IndexerScheduler = new CustomThreadPoolTaskScheduler(50, 100, "Indexing Threads"));
_Indexer.TaskScheduler = _IndexerScheduler;
_Group = new NodesGroup(Configuration.Indexer.Network);
_Disposables.Add(_Group);
_Group.AllowSameGroup = true;
_Group.MaximumNodeConnection = 2;
AddressManager addrman = new AddressManager();
addrman.Add(new NetworkAddress(ip), IPAddress.Parse("127.0.0.1"));
_Group.NodeConnectionParameters.TemplateBehaviors.Add(new AddressManagerBehavior(addrman)
{
Mode = AddressManagerBehaviorMode.None
});
_Group.NodeConnectionParameters.TemplateBehaviors.Add(new ChainBehavior(_Chain)
{
SkipPoWCheck = true
});
_Group.NodeConnectionParameters.TemplateBehaviors.Add(new Behavior(this));
ListenerTrace.Info("Fetching wallet rules...");
_Wallets = _Configuration.Indexer.CreateIndexerClient().GetAllWalletRules();
ListenerTrace.Info("Wallet rules fetched");
ListenerTrace.Info("Fetching wallet subscriptions...");
_Subscriptions = new SubscriptionCollection(_Configuration.GetSubscriptionsTable().Read());
ListenerTrace.Info("Subscriptions fetched");
_Group.Connect();
ListenerTrace.Info("Fetching transactions to broadcast...");
_Disposables.Add(
Configuration
.Topics
.BroadcastedTransactions
.CreateConsumer("listener", true)
.EnsureSubscriptionExists()
.OnMessage((tx, ctl) =>
{
uint256 hash = null;
var repo = Configuration.Indexer.CreateIndexerClient();
var rejects = Configuration.GetRejectTable();
try
{
hash = tx.Transaction.GetHash();
var indexedTx = repo.GetTransaction(hash);
ListenerTrace.Info("Broadcasting " + hash);
var reject = rejects.ReadOne(hash.ToString());
if (reject != null)
{
ListenerTrace.Info("Abort broadcasting of rejected");
return;
}
if (_Broadcasting.Count > 1000)
{
_Broadcasting.Clear();
}
_Broadcasting.TryAdd(hash, tx.Transaction);
if (indexedTx == null || !indexedTx.BlockIds.Any(id => Chain.Contains(id)))
{
var unused = SendMessageAsync(tx.Transaction);
}
var reschedule = new[]
{
TimeSpan.FromMinutes(5),
TimeSpan.FromMinutes(10),
TimeSpan.FromHours(1),
TimeSpan.FromHours(6),
TimeSpan.FromHours(24),
};
if (tx.Tried <= reschedule.Length - 1)
{
ctl.RescheduleIn(reschedule[tx.Tried]);
tx.Tried++;
}
}
catch (Exception ex)
{
if (!_Disposed)
{
LastException = ex;
ListenerTrace.Error("Error for new broadcasted transaction " + hash, ex);
throw;
}
}
}));
ListenerTrace.Info("Transactions to broadcast fetched");
_Disposables.Add(_Configuration
.Topics
.SubscriptionChanges
.EnsureSubscriptionExists()
.AddUnhandledExceptionHandler(ExceptionOnMessagePump)
.OnMessage(c =>
{
using (_SubscriptionSlimLock.LockWrite())
{
if (c.Added)
{
_Subscriptions.Add(c.Subscription);
}
else
{
_Subscriptions.Remove(c.Subscription.Id);
}
}
}));
_Disposables.Add(_Configuration
.Topics
.SendNotifications
.AddUnhandledExceptionHandler(ExceptionOnMessagePump)
.OnMessageAsync((n, act) =>
{
return(SendAsync(n, act).ContinueWith(t =>
{
if (!_Disposed)
{
if (t.Exception != null)
{
LastException = t.Exception;
}
}
}));
}, new OnMessageOptions()
{
MaxConcurrentCalls = 1000,
AutoComplete = true,
AutoRenewTimeout = TimeSpan.Zero
}));
_Disposables.Add(Configuration
.Topics
.AddedAddresses
.CreateConsumer("updater", true)
.EnsureSubscriptionExists()
.AddUnhandledExceptionHandler(ExceptionOnMessagePump)
.OnMessage(evt =>
{
if (evt == null)
{
return;
}
ListenerTrace.Info("New wallet rule");
using (_WalletsSlimLock.LockWrite())
{
foreach (var address in evt)
{
_Wallets.Add(address.CreateWalletRuleEntry());
}
}
}));
}
public void Listen()
{
_Evt.Reset();
_Scheduler = new SingleThreadTaskScheduler();
ListenerTrace.Info("Connecting to node " + Configuration.Indexer.Node + "...");
_Node = _Configuration.Indexer.ConnectToNode(true);
ListenerTrace.Info("Connected");
ListenerTrace.Info("Handshaking...");
_Node.VersionHandshake();
ListenerTrace.Info("Handshaked");
_Chain = new ConcurrentChain(_Configuration.Indexer.Network);
ListenerTrace.Info("Fetching headers...");
_Node.SynchronizeChain(_Chain);
ListenerTrace.Info("Headers fetched tip " + _Chain.Tip.Height);
_Indexer = Configuration.Indexer.CreateIndexer();
ListenerTrace.Info("Indexing indexer chain...");
_Indexer.IndexChain(_Chain);
_Node.MessageReceived += node_MessageReceived;
_Wallets = _Configuration.Indexer.CreateIndexerClient().GetAllWalletRules();
ListenerTrace.Info("Connecting and handshaking for the sender node...");
_SenderNode = _Configuration.Indexer.ConnectToNode(false);
_SenderNode.VersionHandshake();
_SenderNode.MessageReceived += _SenderNode_MessageReceived;
ListenerTrace.Info("Sender node handshaked");
ListenerTrace.Info("Fetching transactions to broadcast...");
_Disposables.Add(
Configuration
.GetBroadcastedTransactionsListenable()
.CreateConsumer()
.EnsureExists()
.OnMessage(evt =>
{
uint256 hash = null;
try
{
if (evt.Addition)
{
var tx = new BroadcastedTransaction(evt.AddedEntity);
hash = tx.Transaction.GetHash();
var value = _BroadcastedTransactions.GetOrAdd(hash, tx);
ListenerTrace.Info("Broadcasting " + hash);
if (value == tx) //Was not present before
{
_SenderNode.SendMessage(new InvPayload(tx.Transaction));
}
}
}
catch (Exception ex)
{
LastException = ex;
ListenerTrace.Error("Error for new broadcasted transaction " + hash, ex);
}
finally
{
DeleteExpiredBroadcasted(evt, hash);
}
}));
ListenerTrace.Info("Transactions to broadcast fetched");
ListenerTrace.Info("Fetching wallet rules...");
_Disposables.Add(Configuration
.GetWalletRuleListenable()
.CreateConsumer()
.EnsureExists()
.OnMessage(evt =>
{
ListenerTrace.Info("New wallet rule");
RunTask("New wallet rule", () =>
{
_Wallets.Add(new WalletRuleEntry(evt.AddedEntity, Configuration.Indexer.CreateIndexerClient()));
}, true);
}));
ListenerTrace.Info("Wallet rules fetched");
var ping = new Timer(Ping, null, 0, 1000 * 60);
_Disposables.Add(ping);
}
static void Main(string[] args)
{
try
{
var options = new IndexerOptions();
if (args.Length == 0)
{
System.Console.WriteLine(options.GetUsage());
}
if (Parser.Default.ParseArguments(args, options))
{
System.Console.WriteLine("NBitcoin.Indexer " + typeof(AzureIndexer).Assembly.GetName().Version);
if (options.All)
{
options.IndexAddresses = true;
options.IndexBlocks = true;
options.IndexWallets = true;
options.IndexChain = true;
options.IndexTransactions = true;
}
var indexer = AzureIndexer.CreateIndexer();
indexer.Configuration.EnsureSetup();
indexer.TaskScheduler = new CustomThreadPoolTaskScheduler(30, 100);
indexer.CheckpointInterval = TimeSpan.Parse(options.CheckpointInterval);
indexer.IgnoreCheckpoints = options.IgnoreCheckpoints;
indexer.FromHeight = options.From;
indexer.ToHeight = options.To;
ChainBase chain = null;
var checkpointRepository = indexer.GetCheckpointRepository();
checkpointRepository.CheckpointSet = null;
if (options.ListCheckpoints)
{
foreach (var checkpoint in checkpointRepository.GetCheckpointsAsync().Result)
{
chain = chain ?? indexer.GetNodeChain();
var fork = chain.FindFork(checkpoint.BlockLocator);
System.Console.WriteLine("Name : " + checkpoint.CheckpointName);
if (fork != null)
{
System.Console.WriteLine("Height : " + fork.Height);
System.Console.WriteLine("Hash : " + fork.HashBlock);
}
System.Console.WriteLine();
}
}
if (options.DeleteCheckpoint != null)
{
checkpointRepository.GetCheckpoint(options.DeleteCheckpoint).DeleteAsync().Wait();
System.Console.WriteLine("Checkpoint " + options.DeleteCheckpoint + " deleted");
}
if (options.AddCheckpoint != null)
{
chain = chain ?? indexer.GetNodeChain();
var split = options.AddCheckpoint.Split(':');
var name = split[0];
var height = int.Parse(split[1]);
var b = chain.GetBlock(height);
var checkpoint = checkpointRepository.GetCheckpoint(name);
checkpoint.SaveProgress(b.GetLocator());
System.Console.WriteLine("Checkpoint " + options.AddCheckpoint + " saved to height " + b.Height);
}
if (ConfigurationManager.AppSettings["MainDirectory"] != null)
{
System.Console.WriteLine("Warning : obsolete appsetting detected, MainDirectory");
string[] oldCheckpoints = new string[] { "transactions", "blocks", "wallets", "balances" };
foreach (var chk in oldCheckpoints)
{
var path = GetFilePath(indexer.Configuration, chk);
if (File.Exists(path))
{
var onlineCheckpoint = checkpointRepository.GetCheckpointsAsync().Result.FirstOrDefault(r => r.CheckpointName.ToLowerInvariant() == chk);
if (onlineCheckpoint == null)
{
onlineCheckpoint = checkpointRepository.GetCheckpoint(indexer.Configuration.CheckpointSetName + "/" + chk);
BlockLocator offlineLocator = new BlockLocator();
offlineLocator.FromBytes(File.ReadAllBytes(path));
onlineCheckpoint.SaveProgress(offlineLocator);
System.Console.WriteLine("Local checkpoint " + chk + " saved in azure");
}
File.Delete(path);
System.Console.WriteLine("Checkpoint File deleted " + path);
}
}
}
if (options.IndexBlocks)
{
chain = chain ?? indexer.GetNodeChain();
indexer.IndexBlocks(chain);
}
if (options.IndexTransactions)
{
chain = chain ?? indexer.GetNodeChain();
indexer.IndexTransactions(chain);
}
if (options.IndexAddresses)
{
chain = chain ?? indexer.GetNodeChain();
indexer.IndexOrderedBalances(chain);
}
if (options.IndexWallets)
{
chain = chain ?? indexer.GetNodeChain();
indexer.IndexWalletBalances(chain);
}
if (options.IndexChain)
{
chain = chain ?? indexer.GetNodeChain();
indexer.IndexChain(chain);
}
}
}
catch (ConfigurationErrorsException ex)
{
System.Console.WriteLine("LocalSettings.config missing settings : " + ex.Message);
}
}
