public async Task <TransactionBatch> NextAsync(CancellationToken token, TimeSpan waitPeriod) { var sw = new Stopwatch(); sw.Start(); TransactionBatch batch = null; while (!_transactionBatchBuffer.TryTake(out batch) && sw.Elapsed <= waitPeriod && !token.IsCancellationRequested) { await Task.Delay(100); } return(batch); }
private async Task PostBatch(CancellationToken token, BlockingCollection <TransactionBatch> transactionBatchBuffer, TransactionBatch transactionBatch) { // If the buffer is full then wait until capacity is available // this ensures we have back pressure to prevent using up all our memory bool added = false; while (!added && !token.IsCancellationRequested) { if (transactionBatchBuffer.TryAdd(transactionBatch)) { added = true; } else { await Task.Delay(100); } } }
private async Task GroupTransactionsAsync(CancellationToken token, List <TableSchema> tableSchemas, Dictionary <string, BlockingCollection <ChangeRecord> > tableChangeBuffers, BlockingCollection <TransactionBatch> transactionBatchBuffer, int transactionBatchSizeLimit) { // create a "one item per table" buffer var currentBuffer = new Dictionary <string, ChangeRecord>(); foreach (var tableSchema in tableSchemas) { currentBuffer.Add(tableSchema.TableName, null); } while (!token.IsCancellationRequested) { // Pull in next values of any table we don't have values for yet or any who previously signalled they had no data at that time var keys = currentBuffer.Keys.ToList(); foreach (var tableKey in keys) { var bufferedValue = currentBuffer[tableKey]; if (bufferedValue == null || bufferedValue.LsnInt == NoDataAvailable) { ChangeRecord change = null; if (tableChangeBuffers[tableKey].TryTake(out change)) { currentBuffer[tableKey] = change; } } } if (currentBuffer.Values.All(x => x == null || x.LsnInt == NoDataAvailable)) { await Task.Delay(1000); continue; } // identify the lowest LSN currently available in our current buffer and a create a new batch for it var currentTransaction = GetEarliestTransactionId(currentBuffer); var transactionBatch = new TransactionBatch(); transactionBatch.Id = currentTransaction; transactionBatch.Part = 1; // Pull data from all table buffers for this LSN var matchingTables = GetTablesOfTransaction(currentBuffer, currentTransaction.LsnInt); foreach (var tableName in matchingTables) { // add the first change of the transaction from our current buffer transactionBatch.Changes.Add(currentBuffer[tableName]); // keep consuming from this table buffer until either the LSN does not match our current LSN // or the table reader signals that we have reached the end of the buffer and no more items will arrive var tableBuffer = tableChangeBuffers[tableName]; bool consume = true; while (consume && !tableBuffer.IsAddingCompleted) { ChangeRecord next = null; if (tableChangeBuffers[tableName].TryTake(out next)) { if (next.LsnInt == currentTransaction.LsnInt) { transactionBatch.Changes.Add(next); // if we have reached the maximum batch size the post and and create another if (transactionBatch.Changes.Count > transactionBatchSizeLimit) { await PostBatch(token, transactionBatchBuffer, transactionBatch); var nextPart = new TransactionBatch(); nextPart.Part = transactionBatch.Part + 1; nextPart.Id = transactionBatch.Id; nextPart.IsMultiPart = true; transactionBatch = nextPart; } } else { // store the value in our current buffer for a future transaction batch currentBuffer[tableName] = next; consume = false; } } else { // there is no data in the buffer but the buffer is not completed. This means that the table reader is slower than us. Wait a little. await Task.Delay(100); } } } // all data has been pulled from the buffers for this transaction // now we post it to the TransactionBatch buffer. await PostBatch(token, transactionBatchBuffer, transactionBatch); } }