/// <summary>
/// All ledger entries that decrease a user's account balance must be procesed here
/// to ensure we don't end up with negative account balances due to race conditions.
///
/// This code runs serially, with a lock on a blob to ensure only one instance is running
/// at a time.
///
/// It is possible that the user can manually add credit to their account while this is running.
/// While this is unlikely, it is also not an issue.
///
/// The most important priority is that the COMMITTED account balance never goes negative.
/// The committed account balance is the sum of all rows in the append-only ledger for an account,
/// and represents real money the user has spend or recieved. If the committed balance goes negative
/// then that implies the user has spent more than they have, and Fifthweek will have to make up the
/// difference (which would be bad).
///
/// The (uncommitted) account balance that the user sees may go negative, and it is desirable
/// for it to do so as it means the user will understand that they will still be billed for
/// money tentatively owed if they top-up again. However if the uncommitted transaction is committed
/// then it will be have to be adjusted by to ensure the committed balance does not go negative.
///
/// To do this, first read the committed account balance for a subscriber when we start processing
/// that subscriber's payments.
///
/// When we commit new payments, subtract the amount from committed balance.
/// If committed balance would be less than zero, adjust the committed payment to ensure account
/// balance is exactly zero.
///
/// This is the only viable option, other than Fifthweek covering the difference or chasing the user
/// for funds. In reality, on the rare occasions this might occur it would be in the
/// order of $0.01. Essentially, we accept that the user cannot pay the amount owed, and
/// do the best we can.
///
/// Afterwards, commit any tips that wouldn't make committed balance less than zero.
/// Keep tips that can't be committed as uncommitted to be processed later if possible.
/// A creator will not see uncommitted tips, but they will count towards the users
/// (uncommitted, posibly negative) account balance. We can always remove old uncommitted
/// tips after a few weeks, to ensure we don't surprise bill the subscriber 2 years later when he
/// logs in again.
///
/// Processing refunds:
/// - For a refund from a particular creator as credits, refund all uncommitted payments.
/// - Ensure this won't be recalculated or committed on next iteration....
/// - We could do this by unsubscribing the user and inserting a refund record
/// which would be taken into account by the payment processing algorithm to zero
/// payments in that week before the refund record.
/// - Creating a refund should be done at the start of payment processing, using the
/// processing timestamp, to ensure the refund is in an uncommitted week.
///
/// - For full refund to the user's bank account, refund all remaining balance by updating
/// ledger (transfer to refund account) and immediately update user's calculated
/// balance (it should be zero).
/// - This must be done within this algorithm to avoid race conditions.
/// - Initiation could be done by the user through the website.
/// - It should be done at the start of this algorithm, to ensure the user gets the refund
/// they expect.
/// - The refund amount, now recorded in the refund account, can then be manually or automatically
/// transferred to the user.
/// - Taxamo needs to be updated.
/// - Tax can also be refunded (moved from user's sales tax account to refund account).
/// </summary>
public async Task ExecuteAsync(IKeepAliveHandler keepAliveHandler, List <PaymentProcessingException> errors, CancellationToken cancellationToken)
{
PaymentsPerformanceLogger.Instance.Clear();
using (PaymentsPerformanceLogger.Instance.Log(typeof(ProcessAllPayments)))
{
keepAliveHandler.AssertNotNull("keepAliveHandler");
errors.AssertNotNull("errors");
var subscriberIds = await this.getAllSubscribers.ExecuteAsync();
// We use the same end time for all subscribers so that when we update
// all account balances at the end the timestamp is accurate.
var endTimeExclusive = this.timestampCreator.Now();
foreach (var subscriberId in subscriberIds)
{
if (cancellationToken.IsCancellationRequested)
{
break;
}
using (PaymentsPerformanceLogger.Instance.Log("Subscriber"))
{
try
{
await this.processPaymentsForSubscriber.ExecuteAsync(
subscriberId,
endTimeExclusive,
keepAliveHandler,
errors);
}
catch (Exception t)
{
errors.Add(new PaymentProcessingException(t, subscriberId, null));
}
}
}
using (PaymentsPerformanceLogger.Instance.Log("BalancesAndCredit"))
{
var updatedBalances = await this.updateAccountBalances.ExecuteAsync(null, endTimeExclusive);
bool recalculateBalances = await this.topUpUserAccountsWithCredit.ExecuteAsync(
updatedBalances, errors, cancellationToken);
if (recalculateBalances)
{
await this.updateAccountBalances.ExecuteAsync(null, endTimeExclusive);
}
}
}
PaymentsPerformanceLogger.Instance.TraceResults();
}
public async Task ExecuteAsync(ILogger logger, IKeepAliveHandler keepAliveHandler, CancellationToken cancellationToken)
{
logger.AssertNotNull("logger");
keepAliveHandler.AssertNotNull("keepAliveHandler");
var stopwatch = new Stopwatch();
stopwatch.Start();
var endTimeExclusive = this.timestampCreator.Now().Subtract(Shared.Constants.GarbageCollectionMinimumAge);
logger.Info("Deleting test user accounts");
await this.deleteTestUserAccounts.ExecuteAsync(endTimeExclusive);
logger.Info("Deleting orphaned files");
var files = await this.getFilesEligibleForGarbageCollection.ExecuteAsync(endTimeExclusive);
foreach (var file in files)
{
if (cancellationToken.IsCancellationRequested)
{
break;
}
Console.Write(".");
try
{
await keepAliveHandler.KeepAliveAsync();
await this.deleteBlobsForFile.ExecuteAsync(file);
await this.deleteFileDbStatement.ExecuteAsync(file.FileId);
}
catch (Exception t)
{
logger.Warn("Failed to delete orphaned file {0}.", file.FileId);
logger.Error(t);
}
}
if (!cancellationToken.IsCancellationRequested)
{
logger.Info("Deleting orphaned blob containers");
await this.deleteOrphanedBlobContainers.ExecuteAsync(logger, keepAliveHandler, endTimeExclusive, cancellationToken);
}
stopwatch.Stop();
logger.Info("Finished garbage collection in {0}s", stopwatch.Elapsed.TotalSeconds);
}
public async Task ExecuteAsync(ILogger logger, IKeepAliveHandler keepAliveHandler, DateTime endTimeExclusive, CancellationToken cancellationToken)
{
logger.AssertNotNull("logger");
keepAliveHandler.AssertNotNull("keepAliveHandler");
var channelIds = await this.getAllChannelIds.ExecuteAsync();
var channelIdsHashSet = new HashSet <Guid>(channelIds.Select(v => v.Value));
// Delete blob containers that parse as Guids but don't correspond to any channel id.
var blobClient = this.cloudStorageAccount.CreateCloudBlobClient();
BlobContinuationToken token = null;
do
{
if (cancellationToken.IsCancellationRequested)
{
break;
}
var segment = await blobClient.ListContainersSegmentedAsync(token);
foreach (var container in segment.Results)
{
Console.Write(".");
if (cancellationToken.IsCancellationRequested)
{
break;
}
await keepAliveHandler.KeepAliveAsync();
Guid channelIdGuid;
if (!Guid.TryParse(container.Name, out channelIdGuid))
{
continue;
}
if (channelIdsHashSet.Contains(channelIdGuid))
{
continue;
}
if (container.Properties.LastModified == null)
{
logger.Warn("Skipping container {0} because last modified date was unavailable", container.Name);
continue;
}
var lastModified = container.Properties.LastModified.Value.UtcDateTime;
if (lastModified >= endTimeExclusive)
{
continue;
}
await container.DeleteAsync();
}
token = segment.ContinuationToken;
}while (token != null);
}
public async Task ExecuteAsync(UserId subscriberId, DateTime endTimeExclusive, IKeepAliveHandler keepAliveHandler, List <PaymentProcessingException> errors)
{
keepAliveHandler.AssertNotNull("keepAliveHandler");
subscriberId.AssertNotNull("subscriberId");
errors.AssertNotNull("errors");
var creators = await this.getCreatorsAndFirstSubscribedDates.ExecuteAsync(subscriberId);
var committedAccountBalanceValue = await this.getCommittedAccountBalanceDbStatement.ExecuteAsync(subscriberId);
if (committedAccountBalanceValue < 0)
{
errors.Add(new PaymentProcessingException(string.Format("Committed account balance was {0} for user {1}.", committedAccountBalanceValue, subscriberId), subscriberId, null));
committedAccountBalanceValue = 0m;
}
var committedAccountBalance = new CommittedAccountBalance(committedAccountBalanceValue);
foreach (var creator in creators)
{
try
{
await keepAliveHandler.KeepAliveAsync();
var latestCommittedLedgerDate = await this.getLatestCommittedLedgerDate.ExecuteAsync(subscriberId, creator.CreatorId);
var startTimeInclusive = latestCommittedLedgerDate ?? PaymentProcessingUtilities.GetPaymentProcessingStartDate(creator.FirstSubscribedDate);
if ((endTimeExclusive - startTimeInclusive) <= MinimumProcessingPeriod)
{
continue;
}
committedAccountBalance = await this.processPaymentsBetweenSubscriberAndCreator.ExecuteAsync(
subscriberId,
creator.CreatorId,
startTimeInclusive,
endTimeExclusive,
committedAccountBalance);
}
catch (Exception t)
{
errors.Add(new PaymentProcessingException(t, subscriberId, creator.CreatorId));
}
}
}
