public Repositories(string connection)
{
#region Data context creating
// ReSharper disable UseObjectOrCollectionInitializer
_Context = new TimezDataContext(connection);
// ReSharper restore UseObjectOrCollectionInitializer
#if DEBUG
_Context.CommandTimeout = 60 * 5;
_Context.Log = new DebuggerWriter();
#endif
#endregion
Boards = BoardsRepository.Create(_Context);
BoardsColors = BoardsColorsRepository.Create(_Context);
BoardsUsers = BoardsUsersRepository.Create(_Context);
Tasks = TasksRepository.Create(_Context);
TasksStatuses = TasksStatusesRepository.Create(_Context);
Users = UsersRepository.Create(_Context);
Organizations = OrganizationsRepository.Create(_Context);
Projects = ProjectsRepository.Create(_Context);
Comments = CommentsRepository.Create(_Context);
EventHistory = EventHistoryRepository.Create(_Context);
Invites = InvitesRepository.Create(_Context);
Texts = TextsRepository.Create(_Context);
}
public FakeEventHistoryService(
EventHistoryRepository eventHistoryRepository,
VersionControl versionControl,
ILogger <EventHistoryService> logger
) : base(eventHistoryRepository, versionControl, logger)
{
}
public void ReportWithdrawalExecuted(WalletWithdrawalEventEntry withdrawal, Action afterMarkedExecuted)
{
// Note: this must occur after withdrawal event processing by this own service
// TODO: try to use VersionControl.WaitForIntegration instead (same functionality)
var retry = true;
while (retry)
{
VersionControl.ExecuteUsingFixedVersion(currentVersion =>
{
if (currentVersion < withdrawal.VersionNumber)
{
return;
}
EventHistoryRepository.Events().FindOneAndUpdate(
eventEntry => eventEntry.Id.Equals(withdrawal.Id),
Builders <EventEntry> .Update.Set(
eventEntry => ((WalletWithdrawalEventEntry)eventEntry).Executed,
true
)
);
afterMarkedExecuted();
_logger.LogInformation(
$"Reported executed withdrawal of {withdrawal.WithdrawalQty} {withdrawal.CoinSymbol}");
retry = false;
});
if (retry)
{
_logger.LogInformation(
$"{nameof(ReportWithdrawalExecuted)} waiting for integration of version number {withdrawal.VersionNumber}");
Task.Delay(1000).Wait();
}
}
}
public EventHistoryService(
EventHistoryRepository eventHistoryRepository,
VersionControl versionControl,
ILogger <EventHistoryService> logger)
{
_logger = logger;
EventHistoryRepository = eventHistoryRepository;
VersionControl = versionControl;
}
/// <summary>
/// </summary>
public UserService(
AccountRepository accountRepository,
EventHistoryRepository eventHistoryRepository,
ILogger <UserService> logger)
{
_logger = logger;
_eventHistoryRepository = eventHistoryRepository;
Accounts = accountRepository.Accounts();
}
public void ReportOverdrawnWithdrawal(WalletWithdrawalEventEntry withdrawal)
{
EventHistoryRepository.Events().FindOneAndUpdate(
eventEntry => eventEntry.Id.Equals(withdrawal.Id),
Builders <EventEntry> .Update.Set(
eventEntry => ((WalletWithdrawalEventEntry)eventEntry).OverdrawnAndCanceledOrders,
true
)
);
}
public void ReportConsolidationValidated(WalletConsolidationTransferEventEntry consolidation, bool validation)
{
_logger.LogInformation(
$"Validation of {consolidation.TransferQty} {consolidation.CoinSymbol} consolidation {(validation ? "successful" : "failed")}");
EventHistoryRepository.Events().FindOneAndUpdate(
eventEntry => eventEntry.Id.Equals(consolidation.Id),
Builders <EventEntry> .Update.Set(
eventEntry => ((WalletConsolidationTransferEventEntry)eventEntry).Valid,
validation
)
);
}
public void ReportConsolidationExecuted(WalletConsolidationTransferEventEntry consolidation)
{
_logger.LogInformation(
$"Reported a new executed consolidation of {consolidation.TransferQty} {consolidation.CoinSymbol}, target becomes {consolidation.NewTargetPublicKeyBalance}");
EventHistoryRepository.Events().FindOneAndUpdate(
eventEntry => eventEntry.Id.Equals(consolidation.Id),
Builders <EventEntry> .Update.Set(
eventEntry => ((WalletConsolidationTransferEventEntry)eventEntry).Executed,
true
)
);
}
public void ReportWithdrawalValidation(WalletWithdrawalEventEntry withdrawal, bool validation)
{
_logger.LogInformation(
$"Validation of {withdrawal.WithdrawalQty} {withdrawal.CoinSymbol} withdrawal {(validation ? "successful" : "failed")}");
EventHistoryRepository.Events().FindOneAndUpdate(
eventEntry => eventEntry.Id.Equals(withdrawal.Id),
Builders <EventEntry> .Update.Set(
eventEntry => ((WalletWithdrawalEventEntry)eventEntry).Validated,
validation
)
);
}
public virtual async Task <IList <EventEntry> > LoadMissingEvents(
long currentVersionNumber,
long?maxVersionNumber = null)
{
// TODO DELETE ME: fast database purge for development purposes
//EventHistoryRepository.Events().DeleteMany(Builders<EventEntry>.Filter.Gt(e => e.VersionNumber, INSERT_NUMBER));
var allNewerEvents = (await EventHistoryRepository
.Events()
.FindAsync(
EventHistoryRepository.VersionAboveFilter(currentVersionNumber, maxVersionNumber)
// ,new FindOptions<EventEntry>
// {
// // The events need to be deterministically sorted by their version number!
// Sort = Builders<EventEntry>.Sort.Ascending(e => e.VersionNumber)
// }
)
).ToList();
var validEvents = allNewerEvents
.GroupBy(e => e.VersionNumber)
.SelectMany(version =>
{
var validEventsOnVersion = new List <EventEntry>();
foreach (var eventEntry in version)
{
validEventsOnVersion.Add(eventEntry);
if (eventEntry is TransactionCommitEventEntry)
{
// Commit found, consider other events invalid
break;
}
}
return(validEventsOnVersion);
})
.ToList();
return(validEvents);
}
/// <summary>
/// Atomically persists multiple event entries representing a single version.
/// </summary>
public virtual async Task <IList <EventEntry> > Persist(
IEnumerable <EventEntry> eventTransaction,
long?alreadyLockedVersionNumber = null)
{
// Copy the list before modifying it
var events = new List <EventEntry>(eventTransaction);
// Take the version number, they all use the same
var versionNumber = events[0].VersionNumber;
// Add the commit event at the end in order to prevent mixing multiple transactions
var commit = new TransactionCommitEventEntry
{
VersionNumber = versionNumber
};
events.Add(commit);
// Double-check validate them all to have the same version number, and assign the same current time
var now = CurrentTime();
foreach (var eventEntry in events)
{
if (eventEntry.VersionNumber != versionNumber)
{
throw new Exception(
$"Integrity error, attempted to persist a transaction consisting of events having different version numbers: expected {versionNumber.ToString()}, actual {eventEntry.VersionNumber.ToString()}");
}
eventEntry.EntryTime = now;
}
// Take the semaphore, so that no other action can use the current version as it's about to change
var versionNumberOutdatedAlready = false;
void InsertLambda(long currentDatabaseVersionNumber)
{
// Make sure version number has not changed, as we can just save ourselves the useless effort otherwise
if (currentDatabaseVersionNumber + 1 != versionNumber)
{
versionNumberOutdatedAlready = true;
return;
}
// We are under synchronization, so we can double-check that we are ahead against other services
if (EventHistoryRepository.Events()
.Find(e => e.VersionNumber > currentDatabaseVersionNumber)
.CountDocuments() != 0)
{
versionNumberOutdatedAlready = true;
return;
}
// Attempt to atomically insert all entries
EventHistoryRepository.Events().InsertMany(events, new InsertManyOptions {
IsOrdered = true
});
}
if (alreadyLockedVersionNumber.HasValue)
{
InsertLambda(alreadyLockedVersionNumber.Value);
}
else
{
VersionControl.ExecuteUsingFixedVersion(InsertLambda);
}
if (versionNumberOutdatedAlready)
{
// Prematurely aborted insertion
_logger.LogError($"Reason for event @ version number {versionNumber} retry: already outdated");
return(null);
}
// Make sure it was inserted with the version number first without other same-versioned concurrent attempts
var foundEventsCursor = await EventHistoryRepository.Events().FindAsync(
EventHistoryRepository.VersionEqFilter(versionNumber)
);
var foundEvents = await foundEventsCursor.ToListAsync();
var failedEntries = new List <EventEntry>();
var foundCommit = false;
var thisSuccessful = false;
foreach (var foundEvent in foundEvents)
{
if (foundCommit)
{
failedEntries.Add(foundEvent);
}
else if (foundEvent is TransactionCommitEventEntry)
{
foundCommit = true;
thisSuccessful = foundEvent.Id.Equals(commit.Id);
}
}
// A nasty workaround to clean up invalid events. They won't be processed, so it's not vital for operation.
// This is not guaranteed to execute though, so TODO change or make another cleanup!
foreach (var failedEntry in failedEntries)
{
_logger.LogError(
$"Note: removing duplicate (uncommitted) failed event entry {failedEntry.GetType().Name} @ version number {versionNumber}");
// Re-written to be sequential, as there were issues with DeleteMany LINQ selector
await EventHistoryRepository.Events().DeleteOneAsync(
e => failedEntry.Id.Equals(e.Id)
);
}
// Return null if the attempted transaction was not the first group of events with the same version number,
// which means it was deemed invalid and then removed
return(thisSuccessful ? events : null);
}
public List <EventEntry> FindByVersionNumber(long versionNumber)
{
return(EventHistoryRepository.Events()
.Find(eventEntry => eventEntry.VersionNumber.Equals(versionNumber))
.ToList());
}
public WalletGenerateEventEntry FindWalletGenerateByPublicKey(string publicKey)
{
return((WalletGenerateEventEntry)EventHistoryRepository.Events().Find(eventEntry =>
eventEntry is WalletGenerateEventEntry &&
((WalletGenerateEventEntry)eventEntry).LastWalletPublicKey.Equals(publicKey)).Single());
}
public EventEntry FindById(ObjectId eventId)
{
return(EventHistoryRepository.Events().Find(eventEntry => eventEntry.Id.Equals(eventId)).Single());
}
public async Task WipeEventsRestoringHotWallets()
{
var walletService = new WebApplicationFactory <WalletService.Startup>().WithWebHostBuilder(builder =>
{
builder.ConfigureTestServices(services =>
{
services.Replace(ServiceDescriptor.Singleton <EventHistoryService>(service =>
new FakeEventHistoryService(
null,
service.GetService <VersionControl>(),
service.GetService <Logger <EventHistoryService> >()
)
)
);
});
});
walletService.CreateClient();
// Wipe the testing DB
var eventHistoryRepository = new EventHistoryRepository(new DataAccess(TestingConnectionString));
eventHistoryRepository.Events().DeleteMany(Builders <EventEntry> .Filter.Where(e => true));
var walletRepo = new WalletRepository(new DataAccess(TestingConnectionString)).HotWallets();
var versionNumber = 1;
walletRepo.Find(e => true)
.ToList()
.ForEach(hotwallet =>
{
Task <decimal> balance;
try
{
balance = AbstractProvider.ProviderLookup[hotwallet.CoinSymbol]
.GetBalance(hotwallet.PublicKey);
balance.Wait(2_000);
if (balance.IsCompletedSuccessfully && balance.Result == 0)
{
walletRepo.DeleteOne(e => e.Id.Equals(hotwallet.Id));
return;
}
}
catch (Exception e)
{
// When we are not sure, we keep the wallet (you can breakpoint this)
var error = e.Message.Trim();
}
walletRepo.UpdateOne(
Builders <HotWallet> .Filter.Eq(e => e.Id, hotwallet.Id),
Builders <HotWallet> .Update.Set(
e => e.CreatedOnVersionNumber, versionNumber)
);
var now = DateTime.Now;
eventHistoryRepository.Events().InsertMany(new EventEntry[]
{
new WalletGenerateEventEntry
{
VersionNumber = versionNumber,
User = hotwallet.User,
AccountId = hotwallet.AccountId,
EntryTime = now,
CoinSymbol = hotwallet.CoinSymbol,
LastWalletPublicKey = hotwallet.PublicKey,
NewSourcePublicKeyBalance = 0,
},
new TransactionCommitEventEntry
{
VersionNumber = versionNumber++,
EntryTime = now,
}
});
});
}
public ConvergenceControllerIntegrationTest(
WebApplicationFactory <Startup> factory)
{
// Start a view service as a direct client, bypassing convergence service proxy middleman
// (This is required because the started client doesn't expose itself - we would have to inject it somehow)
_viewClient = new WebApplicationFactory <XchangeCrypt.Backend.ViewService.Startup>().CreateClient();
// Prepare Convergence Service with injected View Service client
var clientFactory = factory.WithWebHostBuilder(builder =>
{
builder.ConfigureTestServices(services =>
{
// Somehow this works too, even though there are now two services configured
services.AddTransient(service => new ViewProxyService(_viewClient));
services.AddTransient <Logger <ConvergenceControllerIntegrationTest> >();
});
});
_client = clientFactory.CreateClient();
_logger = clientFactory.Server.Host.Services.GetService <Logger <ConvergenceControllerIntegrationTest> >();
_logger.LogInformation("Wiping test DB");
// Wipe the testing DB
_eventHistoryRepository = new EventHistoryRepository(new DataAccess(TestingConnectionString));
_eventHistoryRepository.Events().DeleteMany(Builders <EventEntry> .Filter.Where(e => true));
_logger.LogInformation("Wiped test DB, preparing a new test run");
// Start other queue-based supporting micro-services
_tradingService = new WebApplicationFactory <XchangeCrypt.Backend.TradingService.Startup>()
.WithWebHostBuilder(builder =>
{
builder.ConfigureTestServices(services =>
{
services.Replace(ServiceDescriptor.Singleton <EventHistoryService>(service =>
_tradingServiceEventHistoryService = new TemporalEventHistoryService(
service.GetService <EventHistoryRepository>(),
service.GetService <VersionControl>(),
service.GetService <ILogger <TemporalEventHistoryService> >()
)
));
});
});
_tradingService.CreateClient();
_walletService = new WebApplicationFactory <XchangeCrypt.Backend.WalletService.Startup>()
.WithWebHostBuilder(builder =>
{
builder.ConfigureTestServices(services =>
{
services.Replace(ServiceDescriptor.Singleton <EventHistoryService>(service =>
_walletServiceEventHistoryService = new TemporalEventHistoryService(
service.GetService <EventHistoryRepository>(),
service.GetService <VersionControl>(),
service.GetService <ILogger <TemporalEventHistoryService> >()
)
));
// Replace works the same way as AddSingleton, but it still crashes at
// HostedServiceExecutor#ExecuteAsync with logged NullReferenceException, yet it works properly.
// This crash seems to be fixed by removing the IHostedService replacement,
// letting it receive a replaced injected singleton provider instance.
services.Replace(ServiceDescriptor.Singleton <EthereumProvider>(service =>
_ethProvider = new MockedEthereumProvider(
service.GetService <ILogger <MockedEthereumProvider> >(),
service.GetService <WalletOperationService>(),
service.GetService <EventHistoryService>(),
service.GetService <RandomEntropyService>(),
service.GetService <VersionControl>(),
service.GetService <IConfiguration>()))
);
// services.Replace(ServiceDescriptor.Singleton<IHostedService, EthereumProvider>(
// serviceProvider => serviceProvider.GetService<MockedEthereumProvider>()
// ));
services.Replace(ServiceDescriptor.Singleton <BitcoinProvider>(service =>
_btcProvider = new MockedBitcoinProvider(
service.GetService <ILogger <MockedBitcoinProvider> >(),
service.GetService <WalletOperationService>(),
service.GetService <EventHistoryService>(),
service.GetService <RandomEntropyService>(),
service.GetService <VersionControl>(),
service.GetService <IConfiguration>()))
);
// services.Replace(ServiceDescriptor.Singleton<IHostedService, BitcoinProvider>(
// serviceProvider => serviceProvider.GetService<MockedBitcoinProvider>()
// ));
});
// NOTE: issue: a hosted service doesn't ensure the initialization is done before processing requests!
});
_walletService.CreateClient();
// Make sure this is okay
Assert.Equal(0, _eventHistoryRepository.Events().Find(e => true).CountDocuments());
}
