private static async Task GenerateTweets(AppConfig appConfig, CancellationToken cancellationToken, IProgress <Progress> progress) { WriteLineInColor("Generating tweets...", ConsoleColor.White); Console.WriteLine("Statistics for generated data will be updated for every 60 tweets sent."); // Initialize the telemetry generator: TweetGenerator.Init(); await InitializeCosmosDb(appConfig.CosmosDb.DatabaseId, appConfig.CosmosDb.TweetsContainerId, appConfig.CosmosDb.TweetsPartitionKey); // Find and output the collection details, including # of RU/s. var dataCollection = GetContainerIfExists(appConfig.CosmosDb.DatabaseId, appConfig.CosmosDb.TweetsContainerId); var offer = (OfferV2)_cosmosDbClient.CreateOfferQuery().Where(o => o.ResourceLink == dataCollection.SelfLink).AsEnumerable().FirstOrDefault(); if (offer != null) { var currentCollectionThroughput = offer.Content.OfferThroughput; WriteLineInColor($"Found collection `{appConfig.CosmosDb.TweetsContainerId}` with {currentCollectionThroughput} RU/s ({currentCollectionThroughput} reads/second; {currentCollectionThroughput / 5} writes/second @ 1KB doc size)", ConsoleColor.Green); } // Start sending data to both Event Hubs and Cosmos DB. await SendData(appConfig.CosmosDb.DatabaseId, appConfig.CosmosDb.TweetsContainerId, cancellationToken, progress, 60, 60, true) .ContinueWith(t => { if (t.IsFaulted) { Console.WriteLine($"{t.Exception.Flatten().InnerExceptions}"); } }); }
private static async Task SendData(string databaseId, string containerId, CancellationToken externalCancellationToken, IProgress <Progress> progress, int maxParallelization, int sendNotificationAfter, bool isTweets = false) { // Place Cosmos DB calls into bulkhead to prevent thread starvation caused by failing or waiting calls. // Let any number (int.MaxValue) of calls _queue for an execution slot in the bulkhead to allow the generator to send as many calls as possible. BulkheadPolicy BulkheadForCosmosDbCalls = Policy.BulkheadAsync(maxParallelization, int.MaxValue); if (externalCancellationToken == null) { throw new ArgumentNullException(nameof(externalCancellationToken)); } if (progress == null) { throw new ArgumentNullException(nameof(progress)); } // Perform garbage collection prior to timing for statistics. GC.Collect(); GC.WaitForPendingFinalizers(); var internalCancellationTokenSource = new CancellationTokenSource(); var combinedToken = CancellationTokenSource.CreateLinkedTokenSource(externalCancellationToken, internalCancellationTokenSource.Token).Token; var tasks = new List <Task>(); var messages = new ConcurrentQueue <ColoredMessage>(); var cosmosTimer = new Stopwatch(); // Create the Cosmos DB collection URI: var collectionUri = UriFactory.CreateDocumentCollectionUri(databaseId, containerId); // Ensure none of what follows runs synchronously. await Task.FromResult(true).ConfigureAwait(false); // Continue while cancellation is not requested. while (!combinedToken.IsCancellationRequested) { if (externalCancellationToken.IsCancellationRequested) { return; } for (int i = 0; i <= 4; i++) { _totalMessages++; var thisRequest = _totalMessages; #region Write to Cosmos DB _cosmosRequestsMade++; tasks.Add(BulkheadForCosmosDbCalls.ExecuteAsync(async ct => { try { cosmosTimer.Start(); ResourceResponse <Document> response = null; // Send to Cosmos DB: if (isTweets) { response = await _cosmosDbClient .CreateDocumentAsync(collectionUri, TweetGenerator.Generate()) .ConfigureAwait(false); } else { response = await _cosmosDbClient .CreateDocumentAsync(collectionUri, TelemetryGenerator.Generate()) .ConfigureAwait(false); } cosmosTimer.Stop(); _cosmosElapsedTime = cosmosTimer.ElapsedMilliseconds; // Keep running total of RUs consumed: _cosmosRUsPerBatch += response.RequestCharge; _cosmosRequestsSucceededInBatch++; } catch (DocumentClientException de) { if (!ct.IsCancellationRequested) { messages.Enqueue(new ColoredMessage($"Cosmos DB request {thisRequest} eventually failed with: {de.Message}; Retry-after: {de.RetryAfter.TotalSeconds} seconds.", Color.Red)); } _cosmosRequestsFailed++; } catch (Exception e) { if (!ct.IsCancellationRequested) { messages.Enqueue(new ColoredMessage($"Cosmos DB request {thisRequest} eventually failed with: {e.Message}", Color.Red)); } _cosmosRequestsFailed++; } }, combinedToken) .ContinueWith((t, k) => { if (t.IsFaulted) { messages.Enqueue(new ColoredMessage($"Request to Cosmos DB failed with: {t.Exception?.Flatten().InnerExceptions.First().Message}", Color.Red)); } _cosmosRequestsFailed++; }, thisRequest, TaskContinuationOptions.NotOnRanToCompletion) ); #endregion Write to Cosmos DB if (i == 4 && isTweets) { var span = TimeSpan.FromMilliseconds(2000); await Task.Delay(span, externalCancellationToken); } } if (_totalMessages % sendNotificationAfter == 0) { cosmosTimer.Stop(); _cosmosTotalElapsedTime += _cosmosElapsedTime; _cosmosRequestsSucceeded += _cosmosRequestsSucceededInBatch; // Calculate RUs/second/month: var ruPerSecond = (_cosmosRUsPerBatch / (_cosmosElapsedTime * .001)); var ruPerMonth = ruPerSecond * 86400 * 30; if (!isTweets) { // Add delay every 500 messages that are sent. await Task.Delay(5000, externalCancellationToken); } // Output statistics. Be on the lookout for the following: // - Inserted line shows successful inserts in this batch and throughput for writes/second with RU/s usage and estimated monthly ingestion rate added to Cosmos DB statistics. // - Processing time: Processing time for the past 1,000 requested inserts. // - Total elapsed time: Running total of time taken to process all documents. // - Succeeded shows number of accumulative successful inserts to the service. // - Pending are items in the bulkhead queue. This amount will continue to grow if the service is unable to keep up with demand. // - Accumulative failed requests that encountered an exception. messages.Enqueue(new ColoredMessage($"Total requests: requested {_totalMessages:00} ", Color.Cyan)); messages.Enqueue(new ColoredMessage(string.Empty)); messages.Enqueue(new ColoredMessage($"Inserted {_cosmosRequestsSucceededInBatch:00} docs @ {(_cosmosRequestsSucceededInBatch / (_cosmosElapsedTime * .001)):0.00} writes/s, {ruPerSecond:0.00} RU/s ({(ruPerMonth / (1000 * 1000 * 1000)):0.00}B max monthly 1KB writes) ", Color.White)); messages.Enqueue(new ColoredMessage($"Processing time {_cosmosElapsedTime} ms", Color.Magenta)); messages.Enqueue(new ColoredMessage($"Total elapsed time {(_cosmosTotalElapsedTime * .001):0.00} seconds", Color.Magenta)); messages.Enqueue(new ColoredMessage($"Total succeeded {_cosmosRequestsSucceeded:00} ", Color.Green)); messages.Enqueue(new ColoredMessage($"Total pending {_cosmosRequestsMade - _cosmosRequestsSucceeded - _cosmosRequestsFailed:00} ", Color.Yellow)); messages.Enqueue(new ColoredMessage($"Total failed {_cosmosRequestsFailed:00}", Color.Red)); messages.Enqueue(new ColoredMessage(string.Empty)); // Restart timers and reset batch settings: cosmosTimer.Restart(); _cosmosElapsedTime = 0; _cosmosRUsPerBatch = 0; _cosmosRequestsSucceededInBatch = 0; // Output all messages available right now, in one go. progress.Report(ProgressWithMessages(ConsumeAsEnumerable(messages))); } } messages.Enqueue(new ColoredMessage("Data generation complete", Color.Magenta)); progress.Report(ProgressWithMessages(ConsumeAsEnumerable(messages))); BulkheadForCosmosDbCalls.Dispose(); cosmosTimer.Stop(); }