private static void AddNewAddresses(Datastore dataStore, Address addressData) { var sw = new Stopwatch(); sw.Start(); var bitpayAddresses = addressData.GetOtherBitpayAddresses().Distinct().ToList(); Log.Info("Address {0} had {1} other addresses determined to be Bitpay addresses.", addressData.AddressHash, bitpayAddresses.Count); int newCount = 0, repeatedCount = 0; foreach (var bitpayAddress in bitpayAddresses) { if (!_addressCache.Contains(bitpayAddress)) { Log.Debug("NEW address {0}, adding to analysis queue.", bitpayAddress); _addressCache.Add(bitpayAddress); dataStore.AddAddressToQueue(bitpayAddress); newCount++; } else { Log.Debug("REPEATED address {0}, not adding to queue.", bitpayAddress); repeatedCount++; } } Log.Info("Found {0} new and {1} repeated addresses in address {2}.", newCount, repeatedCount, addressData.AddressHash); sw.Stop(); ConsoleOnly.Debug("Saved addresses in {0}ms", sw.ElapsedMilliseconds); }
public static Address GetAddressData(Datastore datastore, WebClient webClient, AddressToAnalyze address) { try { var sw = new Stopwatch(); sw.Start(); string addressJsonData = webClient.DownloadString(string.Format(AddressUrl, address.Hash)); sw.Stop(); ConsoleOnly.Debug("Downloaded address data in in {0}ms", sw.ElapsedMilliseconds); sw.Reset(); Address addressObject = JsonConvert.DeserializeObject<Address>(addressJsonData); addressObject.IsDirectFind = address.IsDirectFind; addressObject.Transactions = new List<Transaction>(); sw.Start(); foreach (var transactionHash in addressObject.TransactionHashes) { if (!datastore.TransactionExists(transactionHash)) { try { string transactionJsonData = webClient.DownloadString(string.Format(TransactionsUrl, transactionHash)); var transaction = JsonConvert.DeserializeObject<Transaction>(transactionJsonData); addressObject.Transactions.Add(transaction); } catch (WebException we) { Log.Warn(string.Format("WebException while downloading transaction {0}, skipping", transactionHash), we); } } else { ConsoleOnly.Debug("Skipping transaction {0} because it already existed", transactionHash); } } sw.Stop(); ConsoleOnly.Debug("Downloaded transaction data in in {0}ms", sw.ElapsedMilliseconds); return addressObject; } catch (JsonSerializationException e) { Log.Warn(string.Format("Error when deserializing address {0}, skipping.", address), e); SuspiciousAddressLog.Warn(string.Format("Could not deserialize {0}", address), e); return null; } }
private static void Execute() { using (var dataStore = new Datastore(_connectionString)) { int addressesAnalyzed = 0; _originalAddressCount = dataStore.GetUnanalyzedAddressCount(); _originalAnalyzedCount = dataStore.GetAnalyzedAddressCount(); _addressCache = dataStore.GetAllSeenAddresses().ToList(); bool stopping = false; Console.CancelKeyPress += (o, c) => { c.Cancel = true; Log.Warn("Cancel key pressed! Stopping loop..."); stopping = true; }; using (var webClient = new LongerTimeoutWebClient()) { AddressToAnalyze addressToAnalyze; while (dataStore.GetNextAddressToAnalyze(out addressToAnalyze)) { Log.Info("Analyzing address {0}...", addressToAnalyze.Hash); addressesAnalyzed++; var addressData = GetAddressData(dataStore, webClient, addressToAnalyze); if (addressData != null) { try { var sw = new Stopwatch(); sw.Start(); int newTransaction = dataStore.AddAddressData(addressData); sw.Stop(); ConsoleOnly.Debug("Added address data in {0}ms", sw.ElapsedMilliseconds); Log.Info("Found {0} NEW Bitpay transactions out of {1} total.", newTransaction, addressData.Transactions.Count); AddNewAddresses(dataStore, addressData); dataStore.MarkAsAnalyzed(addressToAnalyze.Hash); } catch (SqlException e) { Log.Warn(string.Format("Address {0} FAILED to persist", addressData.AddressHash), e); dataStore.MarkAsAnalyzed(addressToAnalyze.Hash); } } if (stopping) { Log.Debug("Stop requested, breaking from loop."); LogEndData(dataStore, addressesAnalyzed); break; } } } LogEndData(dataStore, addressesAnalyzed); } }
private static void LogEndData(Datastore dataStore, int addressesAnalyzed) { Log.Info("Analyzed {0} addresses this run.", addressesAnalyzed); Log.Info("Found {0} new addresses to be added to the analysis queue.", dataStore.GetUnanalyzedAddressCount() - _originalAnalyzedCount + addressesAnalyzed); }