void LoadCore(Stream stream)
{
JObject obj = JObject.Load(new JsonTextReader(new StreamReader(stream))
{
DateParseHandling = DateParseHandling.DateTimeOffset
});
_Parameters = WalletCreation.FromJson((JObject)obj["Parameters"]);
_PathStates = new Dictionary <KeyPath, PathState>();
if (obj.Property("CurrentIndex") != null) //legacy
{
var idx = (int)(long)obj["CurrentIndex"];
var loadedKeys = (int)(long)obj["LoadedKeys"];
_PathStates.Add(_Parameters.DerivationPath.Derive(0), new PathState()
{
Next = idx,
Loaded = loadedKeys
});
_PathStates.Add(_Parameters.DerivationPath.Derive(1), new PathState()
{
Next = idx,
Loaded = loadedKeys
});
}
var indices = obj["Indices"] as JArray;
if (indices != null)
{
foreach (var indice in indices.OfType <JObject>())
{
_PathStates.Add(KeyPath.Parse((string)indice["KeyPath"]), new PathState()
{
Next = (int)(long)indice["Next"],
Loaded = (int)(long)indice["Loaded"]
});
}
}
_KeyPoolSize = (int)(long)obj["KeyPoolSize"];
Created = (DateTimeOffset)obj["Created"];
_ScanLocation = new BlockLocator();
_ScanLocation.FromBytes(Encoders.Hex.DecodeData((string)obj["Location"]));
_KnownScripts.Clear();
var knownScripts = (JArray)obj["KnownScripts"];
foreach (var known in knownScripts.OfType <JObject>())
{
Script script = Script.FromBytesUnsafe(Encoders.Hex.DecodeData((string)known["ScriptPubKey"]));
if (known["KeyPath"] != null) //Legacy data
{
KeyPath keypath = KeyPath.Parse((string)known["KeyPath"]);
_KnownScripts.Add(script, _Parameters.DerivationPath.Derive(keypath));
}
if (known["AbsoluteKeyPath"] != null)
{
KeyPath keypath = KeyPath.Parse((string)known["AbsoluteKeyPath"]);
_KnownScripts.Add(script, keypath);
}
}
}
public BlockLocator GetIndexProgress()
{
using (var tx = _Engine.GetTransaction())
{
tx.ValuesLazyLoadingIsOn = false;
var existingRow = tx.Select <string, byte[]>("IndexProgress", "");
if (existingRow == null || !existingRow.Exists)
{
return(null);
}
BlockLocator locator = new BlockLocator();
locator.FromBytes(existingRow.Value);
return(locator);
}
}
void LoadCore(Stream stream)
{
JObject obj = JObject.Load(new JsonTextReader(new StreamReader(stream))
{
DateParseHandling = DateParseHandling.DateTimeOffset
});
_CurrentIndex = (int)(long)obj["CurrentIndex"];
_KeyPoolSize = (int)(long)obj["KeyPoolSize"];
_LoadedKeys = (int)(long)obj["LoadedKeys"];
Created = (DateTimeOffset)obj["Created"];
_ScanLocation = new BlockLocator();
_ScanLocation.FromBytes(Encoders.Hex.DecodeData((string)obj["Location"]));
_Parameters = WalletCreation.FromJson((JObject)obj["Parameters"]);
_KnownScripts.Clear();
var knownScripts = (JArray)obj["KnownScripts"];
foreach (var known in knownScripts.OfType <JObject>())
{
Script script = Script.FromBytesUnsafe(Encoders.Hex.DecodeData((string)known["ScriptPubKey"]));
KeyPath keypath = KeyPath.Parse((string)known["KeyPath"]);
_KnownScripts.Add(script, keypath);
}
}
public int Run(ChainBase chain = null)
{
ListenerTrace.Info("Start initial indexing");
int totalProcessed = 0;
using (var node = _Conf.Indexer.ConnectToNode(false))
{
ListenerTrace.Info("Handshaking...");
node.VersionHandshake();
ListenerTrace.Info("Handshaked");
chain = chain ?? node.GetChain();
ListenerTrace.Info("Current chain at height " + chain.Height);
var blockRepository = new NodeBlocksRepository(node);
var blobLock = GetInitBlob();
string lease = null;
try
{
blobLock.UploadText("Enqueuing");
lease = blobLock.AcquireLease(null, null);
}
catch (StorageException)
{
}
if (lease != null)
{
ListenerTrace.Info("Queueing index jobs");
EnqueueJobs(blockRepository, chain, blobLock, lease);
}
ListenerTrace.Info("Dequeuing index jobs");
while (true)
{
var msg = _Conf.Topics
.InitialIndexing
.ReceiveAsync(TimeSpan.FromMilliseconds(1000))
.Result;
var ns = _Conf.Topics.InitialIndexing.GetNamespace();
var description = ns.GetQueue(_Conf.Topics.InitialIndexing.Queue);
Console.WriteLine("Work remaining in the queue : " + description.MessageCountDetails.ActiveMessageCount);
if (msg == null)
{
var state = blobLock.DownloadText();
if (state == "Enqueuing" || description.MessageCountDetails.ActiveMessageCount != 0)
{
ListenerTrace.Info("Additional work will be enqueued...");
continue;
}
else
{
var locator = new BlockLocator();
locator.FromBytes(Encoders.Hex.DecodeData(state));
UpdateCheckpoints(locator);
break;
}
}
using (msg.Message)
{
var range = msg.Body;
using (var sched = new CustomThreadPoolTaskScheduler(50, 100, range.ToString()))
{
ListenerTrace.Info("Processing " + range.ToString());
totalProcessed++;
var task = _IndexTasks[range.Target];
BlockFetcher fetcher = new BlockFetcher(task.Item1, blockRepository, chain)
{
FromHeight = range.From,
ToHeight = range.From + range.Count - 1
};
try
{
task.Item2.SaveProgression = false;
task.Item2.EnsureIsSetup = totalProcessed == 0;
var index = Task.Factory.StartNew(() =>
{
task.Item2.Index(fetcher, sched);
}, TaskCreationOptions.LongRunning);
while (!index.Wait(TimeSpan.FromMinutes(4)))
{
msg.Message.RenewLock();
ListenerTrace.Info("Lock renewed");
}
}
catch (AggregateException aex)
{
ExceptionDispatchInfo.Capture(aex.InnerException).Throw();
throw;
}
range.Processed = true;
msg.Message.Complete();
}
}
}
}
ListenerTrace.Info("Initial indexing terminated");
return(totalProcessed);
}
void LoadCore(Stream stream)
{
JObject obj = JObject.Load(new JsonTextReader(new StreamReader(stream))
{
DateParseHandling = DateParseHandling.DateTimeOffset
});
_Parameters = WalletCreation.FromJson((JObject)obj["Parameters"]);
_PathStates = new Dictionary<KeyPath, PathState>();
if(obj.Property("CurrentIndex") != null) //legacy
{
var idx = (int)(long)obj["CurrentIndex"];
var loadedKeys = (int)(long)obj["LoadedKeys"];
_PathStates.Add(_Parameters.DerivationPath.Derive(0), new PathState()
{
Next = idx,
Loaded = loadedKeys
});
_PathStates.Add(_Parameters.DerivationPath.Derive(1), new PathState()
{
Next = idx,
Loaded = loadedKeys
});
}
var indices = obj["Indices"] as JArray;
if(indices != null)
{
foreach(var indice in indices.OfType<JObject>())
{
_PathStates.Add(KeyPath.Parse((string)indice["KeyPath"]), new PathState()
{
Next = (int)(long)indice["Next"],
Loaded = (int)(long)indice["Loaded"]
});
}
}
_KeyPoolSize = (int)(long)obj["KeyPoolSize"];
Created = (DateTimeOffset)obj["Created"];
_ScanLocation = new BlockLocator();
_ScanLocation.FromBytes(Encoders.Hex.DecodeData((string)obj["Location"]));
_KnownScripts.Clear();
var knownScripts = (JArray)obj["KnownScripts"];
foreach(var known in knownScripts.OfType<JObject>())
{
Script script = Script.FromBytesUnsafe(Encoders.Hex.DecodeData((string)known["ScriptPubKey"]));
if(known["KeyPath"] != null) //Legacy data
{
KeyPath keypath = KeyPath.Parse((string)known["KeyPath"]);
_KnownScripts.Add(script, _Parameters.DerivationPath.Derive(keypath));
}
if(known["AbsoluteKeyPath"] != null)
{
KeyPath keypath = KeyPath.Parse((string)known["AbsoluteKeyPath"]);
_KnownScripts.Add(script, keypath);
}
}
}
void LoadCore(Stream stream)
{
JObject obj = JObject.Load(new JsonTextReader(new StreamReader(stream))
{
DateParseHandling = DateParseHandling.DateTimeOffset
});
_CurrentIndex = (int)(long)obj["CurrentIndex"];
_KeyPoolSize = (int)(long)obj["KeyPoolSize"];
_LoadedKeys = (int)(long)obj["LoadedKeys"];
Created = (DateTimeOffset)obj["Created"];
_ScanLocation = new BlockLocator();
_ScanLocation.FromBytes(Encoders.Hex.DecodeData((string)obj["Location"]));
_Parameters = WalletCreation.FromJson((JObject)obj["Parameters"]);
_KnownScripts.Clear();
var knownScripts = (JArray)obj["KnownScripts"];
foreach(var known in knownScripts.OfType<JObject>())
{
Script script = Script.FromBytesUnsafe(Encoders.Hex.DecodeData((string)known["ScriptPubKey"]));
KeyPath keypath = KeyPath.Parse((string)known["KeyPath"]);
_KnownScripts.Add(script, keypath);
}
}
static void Main(string[] args)
{
try
{
var options = new IndexerOptions();
if (args.Length == 0)
System.Console.WriteLine(options.GetUsage());
if (Parser.Default.ParseArguments(args, options))
{
System.Console.WriteLine("NBitcoin.Indexer " + typeof(AzureIndexer).Assembly.GetName().Version);
if (options.All)
{
options.IndexAddresses = true;
options.IndexBlocks = true;
options.IndexWallets = true;
options.IndexChain = true;
options.IndexTransactions = true;
}
var indexer = AzureIndexer.CreateIndexer();
indexer.Configuration.EnsureSetup();
indexer.TaskScheduler = new CustomThreadPoolTaskScheduler(30, 100);
indexer.CheckpointInterval = TimeSpan.Parse(options.CheckpointInterval);
indexer.IgnoreCheckpoints = options.IgnoreCheckpoints;
indexer.FromHeight = options.From;
indexer.ToHeight = options.To;
ChainBase chain = null;
var checkpointRepository = indexer.GetCheckpointRepository();
checkpointRepository.CheckpointSet = null;
if (options.ListCheckpoints)
{
foreach (var checkpoint in checkpointRepository.GetCheckpointsAsync().Result)
{
chain = chain ?? indexer.GetNodeChain();
var fork = chain.FindFork(checkpoint.BlockLocator);
System.Console.WriteLine("Name : " + checkpoint.CheckpointName);
if (fork != null)
{
System.Console.WriteLine("Height : " + fork.Height);
System.Console.WriteLine("Hash : " + fork.HashBlock);
}
System.Console.WriteLine();
}
}
if (options.DeleteCheckpoint != null)
{
checkpointRepository.GetCheckpoint(options.DeleteCheckpoint).DeleteAsync().Wait();
System.Console.WriteLine("Checkpoint " + options.DeleteCheckpoint + " deleted");
}
if (options.AddCheckpoint != null)
{
chain = chain ?? indexer.GetNodeChain();
var split = options.AddCheckpoint.Split(':');
var name = split[0];
var height = int.Parse(split[1]);
var b = chain.GetBlock(height);
var checkpoint = checkpointRepository.GetCheckpoint(name);
checkpoint.SaveProgress(b.GetLocator());
System.Console.WriteLine("Checkpoint " + options.AddCheckpoint + " saved to height " + b.Height);
}
if (ConfigurationManager.AppSettings["MainDirectory"] != null)
{
System.Console.WriteLine("Warning : obsolete appsetting detected, MainDirectory");
string[] oldCheckpoints = new string[] { "transactions", "blocks", "wallets", "balances" };
foreach (var chk in oldCheckpoints)
{
var path = GetFilePath(indexer.Configuration, chk);
if (File.Exists(path))
{
var onlineCheckpoint = checkpointRepository.GetCheckpointsAsync().Result.FirstOrDefault(r => r.CheckpointName.ToLowerInvariant() == chk);
if (onlineCheckpoint == null)
{
onlineCheckpoint = checkpointRepository.GetCheckpoint(indexer.Configuration.CheckpointSetName + "/" + chk);
BlockLocator offlineLocator = new BlockLocator();
offlineLocator.FromBytes(File.ReadAllBytes(path));
onlineCheckpoint.SaveProgress(offlineLocator);
System.Console.WriteLine("Local checkpoint " + chk + " saved in azure");
}
File.Delete(path);
System.Console.WriteLine("Checkpoint File deleted " + path);
}
}
}
if (options.IndexBlocks)
{
chain = chain ?? indexer.GetNodeChain();
indexer.IndexBlocks(chain);
}
if (options.IndexTransactions)
{
chain = chain ?? indexer.GetNodeChain();
indexer.IndexTransactions(chain);
}
if (options.IndexAddresses)
{
chain = chain ?? indexer.GetNodeChain();
indexer.IndexOrderedBalances(chain);
}
if (options.IndexWallets)
{
chain = chain ?? indexer.GetNodeChain();
indexer.IndexWalletBalances(chain);
}
if (options.IndexChain)
{
chain = chain ?? indexer.GetNodeChain();
indexer.IndexChain(chain);
}
}
}
catch (ConfigurationErrorsException ex)
{
System.Console.WriteLine("LocalSettings.config missing settings : " + ex.Message);
}
}
static void Main(string[] args)
{
try
{
var options = new IndexerOptions();
if (args.Length == 0)
{
System.Console.WriteLine(options.GetUsage());
}
if (Parser.Default.ParseArguments(args, options))
{
System.Console.WriteLine("NBitcoin.Indexer " + typeof(AzureIndexer).Assembly.GetName().Version);
if (options.All)
{
options.IndexAddresses = true;
options.IndexBlocks = true;
options.IndexWallets = true;
options.IndexChain = true;
options.IndexTransactions = true;
}
var indexer = AzureIndexer.CreateIndexer();
indexer.Configuration.EnsureSetup();
indexer.TaskScheduler = new CustomThreadPoolTaskScheduler(30, 100);
indexer.CheckpointInterval = TimeSpan.Parse(options.CheckpointInterval);
indexer.IgnoreCheckpoints = options.IgnoreCheckpoints;
indexer.FromHeight = options.From;
indexer.ToHeight = options.To;
ChainBase chain = null;
var checkpointRepository = indexer.GetCheckpointRepository();
checkpointRepository.CheckpointSet = null;
if (options.ListCheckpoints)
{
foreach (var checkpoint in checkpointRepository.GetCheckpointsAsync().Result)
{
chain = chain ?? indexer.GetNodeChain();
var fork = chain.FindFork(checkpoint.BlockLocator);
System.Console.WriteLine("Name : " + checkpoint.CheckpointName);
if (fork != null)
{
System.Console.WriteLine("Height : " + fork.Height);
System.Console.WriteLine("Hash : " + fork.HashBlock);
}
System.Console.WriteLine();
}
}
if (options.DeleteCheckpoint != null)
{
checkpointRepository.GetCheckpoint(options.DeleteCheckpoint).DeleteAsync().Wait();
System.Console.WriteLine("Checkpoint " + options.DeleteCheckpoint + " deleted");
}
if (options.AddCheckpoint != null)
{
chain = chain ?? indexer.GetNodeChain();
var split = options.AddCheckpoint.Split(':');
var name = split[0];
var height = int.Parse(split[1]);
var b = chain.GetBlock(height);
var checkpoint = checkpointRepository.GetCheckpoint(name);
checkpoint.SaveProgress(b.GetLocator());
System.Console.WriteLine("Checkpoint " + options.AddCheckpoint + " saved to height " + b.Height);
}
if (ConfigurationManager.AppSettings["MainDirectory"] != null)
{
System.Console.WriteLine("Warning : obsolete appsetting detected, MainDirectory");
string[] oldCheckpoints = new string[] { "transactions", "blocks", "wallets", "balances" };
foreach (var chk in oldCheckpoints)
{
var path = GetFilePath(indexer.Configuration, chk);
if (File.Exists(path))
{
var onlineCheckpoint = checkpointRepository.GetCheckpointsAsync().Result.FirstOrDefault(r => r.CheckpointName.ToLowerInvariant() == chk);
if (onlineCheckpoint == null)
{
onlineCheckpoint = checkpointRepository.GetCheckpoint(indexer.Configuration.CheckpointSetName + "/" + chk);
BlockLocator offlineLocator = new BlockLocator();
offlineLocator.FromBytes(File.ReadAllBytes(path));
onlineCheckpoint.SaveProgress(offlineLocator);
System.Console.WriteLine("Local checkpoint " + chk + " saved in azure");
}
File.Delete(path);
System.Console.WriteLine("Checkpoint File deleted " + path);
}
}
}
if (options.IndexBlocks)
{
chain = chain ?? indexer.GetNodeChain();
indexer.IndexBlocks(chain);
}
if (options.IndexTransactions)
{
chain = chain ?? indexer.GetNodeChain();
indexer.IndexTransactions(chain);
}
if (options.IndexAddresses)
{
chain = chain ?? indexer.GetNodeChain();
indexer.IndexOrderedBalances(chain);
}
if (options.IndexWallets)
{
chain = chain ?? indexer.GetNodeChain();
indexer.IndexWalletBalances(chain);
}
if (options.IndexChain)
{
chain = chain ?? indexer.GetNodeChain();
indexer.IndexChain(chain);
}
}
}
catch (ConfigurationErrorsException ex)
{
System.Console.WriteLine("LocalSettings.config missing settings : " + ex.Message);
}
}