public void ReportWithdrawalExecuted(WalletWithdrawalEventEntry withdrawal, Action afterMarkedExecuted) { // Note: this must occur after withdrawal event processing by this own service // TODO: try to use VersionControl.WaitForIntegration instead (same functionality) var retry = true; while (retry) { VersionControl.ExecuteUsingFixedVersion(currentVersion => { if (currentVersion < withdrawal.VersionNumber) { return; } EventHistoryRepository.Events().FindOneAndUpdate( eventEntry => eventEntry.Id.Equals(withdrawal.Id), Builders <EventEntry> .Update.Set( eventEntry => ((WalletWithdrawalEventEntry)eventEntry).Executed, true ) ); afterMarkedExecuted(); _logger.LogInformation( $"Reported executed withdrawal of {withdrawal.WithdrawalQty} {withdrawal.CoinSymbol}"); retry = false; }); if (retry) { _logger.LogInformation( $"{nameof(ReportWithdrawalExecuted)} waiting for integration of version number {withdrawal.VersionNumber}"); Task.Delay(1000).Wait(); } } }
public void ReportOverdrawnWithdrawal(WalletWithdrawalEventEntry withdrawal) { EventHistoryRepository.Events().FindOneAndUpdate( eventEntry => eventEntry.Id.Equals(withdrawal.Id), Builders <EventEntry> .Update.Set( eventEntry => ((WalletWithdrawalEventEntry)eventEntry).OverdrawnAndCanceledOrders, true ) ); }
public void ReportConsolidationValidated(WalletConsolidationTransferEventEntry consolidation, bool validation) { _logger.LogInformation( $"Validation of {consolidation.TransferQty} {consolidation.CoinSymbol} consolidation {(validation ? "successful" : "failed")}"); EventHistoryRepository.Events().FindOneAndUpdate( eventEntry => eventEntry.Id.Equals(consolidation.Id), Builders <EventEntry> .Update.Set( eventEntry => ((WalletConsolidationTransferEventEntry)eventEntry).Valid, validation ) ); }
public void ReportConsolidationExecuted(WalletConsolidationTransferEventEntry consolidation) { _logger.LogInformation( $"Reported a new executed consolidation of {consolidation.TransferQty} {consolidation.CoinSymbol}, target becomes {consolidation.NewTargetPublicKeyBalance}"); EventHistoryRepository.Events().FindOneAndUpdate( eventEntry => eventEntry.Id.Equals(consolidation.Id), Builders <EventEntry> .Update.Set( eventEntry => ((WalletConsolidationTransferEventEntry)eventEntry).Executed, true ) ); }
public void ReportWithdrawalValidation(WalletWithdrawalEventEntry withdrawal, bool validation) { _logger.LogInformation( $"Validation of {withdrawal.WithdrawalQty} {withdrawal.CoinSymbol} withdrawal {(validation ? "successful" : "failed")}"); EventHistoryRepository.Events().FindOneAndUpdate( eventEntry => eventEntry.Id.Equals(withdrawal.Id), Builders <EventEntry> .Update.Set( eventEntry => ((WalletWithdrawalEventEntry)eventEntry).Validated, validation ) ); }
private void Integrate(Action action, int events = 1) { var oldVersion = _eventHistoryRepository .Events() .Find(e => true) .SortByDescending(e => e.VersionNumber) .FirstOrDefault() ?.VersionNumber ?? 0; action(); _tradingServiceEventHistoryService.VersionControl.WaitForIntegration(oldVersion + events); _walletServiceEventHistoryService.VersionControl.WaitForIntegration(oldVersion + events); }
public virtual async Task <IList <EventEntry> > LoadMissingEvents( long currentVersionNumber, long?maxVersionNumber = null) { // TODO DELETE ME: fast database purge for development purposes //EventHistoryRepository.Events().DeleteMany(Builders<EventEntry>.Filter.Gt(e => e.VersionNumber, INSERT_NUMBER)); var allNewerEvents = (await EventHistoryRepository .Events() .FindAsync( EventHistoryRepository.VersionAboveFilter(currentVersionNumber, maxVersionNumber) // ,new FindOptions<EventEntry> // { // // The events need to be deterministically sorted by their version number! // Sort = Builders<EventEntry>.Sort.Ascending(e => e.VersionNumber) // } ) ).ToList(); var validEvents = allNewerEvents .GroupBy(e => e.VersionNumber) .SelectMany(version => { var validEventsOnVersion = new List <EventEntry>(); foreach (var eventEntry in version) { validEventsOnVersion.Add(eventEntry); if (eventEntry is TransactionCommitEventEntry) { // Commit found, consider other events invalid break; } } return(validEventsOnVersion); }) .ToList(); return(validEvents); }
/// <summary> /// Atomically persists multiple event entries representing a single version. /// </summary> public virtual async Task <IList <EventEntry> > Persist( IEnumerable <EventEntry> eventTransaction, long?alreadyLockedVersionNumber = null) { // Copy the list before modifying it var events = new List <EventEntry>(eventTransaction); // Take the version number, they all use the same var versionNumber = events[0].VersionNumber; // Add the commit event at the end in order to prevent mixing multiple transactions var commit = new TransactionCommitEventEntry { VersionNumber = versionNumber }; events.Add(commit); // Double-check validate them all to have the same version number, and assign the same current time var now = CurrentTime(); foreach (var eventEntry in events) { if (eventEntry.VersionNumber != versionNumber) { throw new Exception( $"Integrity error, attempted to persist a transaction consisting of events having different version numbers: expected {versionNumber.ToString()}, actual {eventEntry.VersionNumber.ToString()}"); } eventEntry.EntryTime = now; } // Take the semaphore, so that no other action can use the current version as it's about to change var versionNumberOutdatedAlready = false; void InsertLambda(long currentDatabaseVersionNumber) { // Make sure version number has not changed, as we can just save ourselves the useless effort otherwise if (currentDatabaseVersionNumber + 1 != versionNumber) { versionNumberOutdatedAlready = true; return; } // We are under synchronization, so we can double-check that we are ahead against other services if (EventHistoryRepository.Events() .Find(e => e.VersionNumber > currentDatabaseVersionNumber) .CountDocuments() != 0) { versionNumberOutdatedAlready = true; return; } // Attempt to atomically insert all entries EventHistoryRepository.Events().InsertMany(events, new InsertManyOptions { IsOrdered = true }); } if (alreadyLockedVersionNumber.HasValue) { InsertLambda(alreadyLockedVersionNumber.Value); } else { VersionControl.ExecuteUsingFixedVersion(InsertLambda); } if (versionNumberOutdatedAlready) { // Prematurely aborted insertion _logger.LogError($"Reason for event @ version number {versionNumber} retry: already outdated"); return(null); } // Make sure it was inserted with the version number first without other same-versioned concurrent attempts var foundEventsCursor = await EventHistoryRepository.Events().FindAsync( EventHistoryRepository.VersionEqFilter(versionNumber) ); var foundEvents = await foundEventsCursor.ToListAsync(); var failedEntries = new List <EventEntry>(); var foundCommit = false; var thisSuccessful = false; foreach (var foundEvent in foundEvents) { if (foundCommit) { failedEntries.Add(foundEvent); } else if (foundEvent is TransactionCommitEventEntry) { foundCommit = true; thisSuccessful = foundEvent.Id.Equals(commit.Id); } } // A nasty workaround to clean up invalid events. They won't be processed, so it's not vital for operation. // This is not guaranteed to execute though, so TODO change or make another cleanup! foreach (var failedEntry in failedEntries) { _logger.LogError( $"Note: removing duplicate (uncommitted) failed event entry {failedEntry.GetType().Name} @ version number {versionNumber}"); // Re-written to be sequential, as there were issues with DeleteMany LINQ selector await EventHistoryRepository.Events().DeleteOneAsync( e => failedEntry.Id.Equals(e.Id) ); } // Return null if the attempted transaction was not the first group of events with the same version number, // which means it was deemed invalid and then removed return(thisSuccessful ? events : null); }
public List <EventEntry> FindByVersionNumber(long versionNumber) { return(EventHistoryRepository.Events() .Find(eventEntry => eventEntry.VersionNumber.Equals(versionNumber)) .ToList()); }
public WalletGenerateEventEntry FindWalletGenerateByPublicKey(string publicKey) { return((WalletGenerateEventEntry)EventHistoryRepository.Events().Find(eventEntry => eventEntry is WalletGenerateEventEntry && ((WalletGenerateEventEntry)eventEntry).LastWalletPublicKey.Equals(publicKey)).Single()); }
public EventEntry FindById(ObjectId eventId) { return(EventHistoryRepository.Events().Find(eventEntry => eventEntry.Id.Equals(eventId)).Single()); }
public async Task WipeEventsRestoringHotWallets() { var walletService = new WebApplicationFactory <WalletService.Startup>().WithWebHostBuilder(builder => { builder.ConfigureTestServices(services => { services.Replace(ServiceDescriptor.Singleton <EventHistoryService>(service => new FakeEventHistoryService( null, service.GetService <VersionControl>(), service.GetService <Logger <EventHistoryService> >() ) ) ); }); }); walletService.CreateClient(); // Wipe the testing DB var eventHistoryRepository = new EventHistoryRepository(new DataAccess(TestingConnectionString)); eventHistoryRepository.Events().DeleteMany(Builders <EventEntry> .Filter.Where(e => true)); var walletRepo = new WalletRepository(new DataAccess(TestingConnectionString)).HotWallets(); var versionNumber = 1; walletRepo.Find(e => true) .ToList() .ForEach(hotwallet => { Task <decimal> balance; try { balance = AbstractProvider.ProviderLookup[hotwallet.CoinSymbol] .GetBalance(hotwallet.PublicKey); balance.Wait(2_000); if (balance.IsCompletedSuccessfully && balance.Result == 0) { walletRepo.DeleteOne(e => e.Id.Equals(hotwallet.Id)); return; } } catch (Exception e) { // When we are not sure, we keep the wallet (you can breakpoint this) var error = e.Message.Trim(); } walletRepo.UpdateOne( Builders <HotWallet> .Filter.Eq(e => e.Id, hotwallet.Id), Builders <HotWallet> .Update.Set( e => e.CreatedOnVersionNumber, versionNumber) ); var now = DateTime.Now; eventHistoryRepository.Events().InsertMany(new EventEntry[] { new WalletGenerateEventEntry { VersionNumber = versionNumber, User = hotwallet.User, AccountId = hotwallet.AccountId, EntryTime = now, CoinSymbol = hotwallet.CoinSymbol, LastWalletPublicKey = hotwallet.PublicKey, NewSourcePublicKeyBalance = 0, }, new TransactionCommitEventEntry { VersionNumber = versionNumber++, EntryTime = now, } }); }); }
public ConvergenceControllerIntegrationTest( WebApplicationFactory <Startup> factory) { // Start a view service as a direct client, bypassing convergence service proxy middleman // (This is required because the started client doesn't expose itself - we would have to inject it somehow) _viewClient = new WebApplicationFactory <XchangeCrypt.Backend.ViewService.Startup>().CreateClient(); // Prepare Convergence Service with injected View Service client var clientFactory = factory.WithWebHostBuilder(builder => { builder.ConfigureTestServices(services => { // Somehow this works too, even though there are now two services configured services.AddTransient(service => new ViewProxyService(_viewClient)); services.AddTransient <Logger <ConvergenceControllerIntegrationTest> >(); }); }); _client = clientFactory.CreateClient(); _logger = clientFactory.Server.Host.Services.GetService <Logger <ConvergenceControllerIntegrationTest> >(); _logger.LogInformation("Wiping test DB"); // Wipe the testing DB _eventHistoryRepository = new EventHistoryRepository(new DataAccess(TestingConnectionString)); _eventHistoryRepository.Events().DeleteMany(Builders <EventEntry> .Filter.Where(e => true)); _logger.LogInformation("Wiped test DB, preparing a new test run"); // Start other queue-based supporting micro-services _tradingService = new WebApplicationFactory <XchangeCrypt.Backend.TradingService.Startup>() .WithWebHostBuilder(builder => { builder.ConfigureTestServices(services => { services.Replace(ServiceDescriptor.Singleton <EventHistoryService>(service => _tradingServiceEventHistoryService = new TemporalEventHistoryService( service.GetService <EventHistoryRepository>(), service.GetService <VersionControl>(), service.GetService <ILogger <TemporalEventHistoryService> >() ) )); }); }); _tradingService.CreateClient(); _walletService = new WebApplicationFactory <XchangeCrypt.Backend.WalletService.Startup>() .WithWebHostBuilder(builder => { builder.ConfigureTestServices(services => { services.Replace(ServiceDescriptor.Singleton <EventHistoryService>(service => _walletServiceEventHistoryService = new TemporalEventHistoryService( service.GetService <EventHistoryRepository>(), service.GetService <VersionControl>(), service.GetService <ILogger <TemporalEventHistoryService> >() ) )); // Replace works the same way as AddSingleton, but it still crashes at // HostedServiceExecutor#ExecuteAsync with logged NullReferenceException, yet it works properly. // This crash seems to be fixed by removing the IHostedService replacement, // letting it receive a replaced injected singleton provider instance. services.Replace(ServiceDescriptor.Singleton <EthereumProvider>(service => _ethProvider = new MockedEthereumProvider( service.GetService <ILogger <MockedEthereumProvider> >(), service.GetService <WalletOperationService>(), service.GetService <EventHistoryService>(), service.GetService <RandomEntropyService>(), service.GetService <VersionControl>(), service.GetService <IConfiguration>())) ); // services.Replace(ServiceDescriptor.Singleton<IHostedService, EthereumProvider>( // serviceProvider => serviceProvider.GetService<MockedEthereumProvider>() // )); services.Replace(ServiceDescriptor.Singleton <BitcoinProvider>(service => _btcProvider = new MockedBitcoinProvider( service.GetService <ILogger <MockedBitcoinProvider> >(), service.GetService <WalletOperationService>(), service.GetService <EventHistoryService>(), service.GetService <RandomEntropyService>(), service.GetService <VersionControl>(), service.GetService <IConfiguration>())) ); // services.Replace(ServiceDescriptor.Singleton<IHostedService, BitcoinProvider>( // serviceProvider => serviceProvider.GetService<MockedBitcoinProvider>() // )); }); // NOTE: issue: a hosted service doesn't ensure the initialization is done before processing requests! }); _walletService.CreateClient(); // Make sure this is okay Assert.Equal(0, _eventHistoryRepository.Events().Find(e => true).CountDocuments()); }