// Send data to Cosmos DB and Event Hub: private static async Task SendData(int randomSeed, int waittime, CancellationToken externalCancellationToken, IProgress <Progress> progress) { if (waittime > 0) { var span = TimeSpan.FromMilliseconds(waittime); await Task.Delay(span, externalCancellationToken); } if (externalCancellationToken == null) { throw new ArgumentNullException(nameof(externalCancellationToken)); } if (progress == null) { throw new ArgumentNullException(nameof(progress)); } // Perform garbage collection prior to timing for statistics. GC.Collect(); GC.WaitForPendingFinalizers(); var internalCancellationTokenSource = new CancellationTokenSource(); var combinedToken = CancellationTokenSource.CreateLinkedTokenSource(externalCancellationToken, internalCancellationTokenSource.Token).Token; var random = new Random(randomSeed); var tasks = new List <Task>(); var messages = new ConcurrentQueue <ColoredMessage>(); var eventHubsTimer = new Stopwatch(); var cosmosTimer = new Stopwatch(); // Create the Cosmos DB collection URI: var collectionUri = UriFactory.CreateDocumentCollectionUri(DatabaseName, CollectionName); // Ensure none of what follows runs synchronously. await Task.FromResult(true).ConfigureAwait(false); // Continue while cancellation is not requested. while (!combinedToken.IsCancellationRequested) { if (externalCancellationToken.IsCancellationRequested) { return; } _totalMessages++; var thisRequest = _totalMessages; var carEvent = TelemetryGenerator.GenerateMessage(); #region Write to Cosmos DB _cosmosRequestsMade++; tasks.Add(BulkheadForCosmosDbCalls.ExecuteAsync(async ct => { try { cosmosTimer.Start(); // Send to Cosmos DB: var response = await _cosmosDbClient.CreateDocumentAsync(collectionUri, carEvent) .ConfigureAwait(false); cosmosTimer.Stop(); _cosmosElapsedTime = cosmosTimer.ElapsedMilliseconds; // Keep running total of RUs consumed: _cosmosRUsPerBatch += response.RequestCharge; _cosmosRequestsSucceededInBatch++; } catch (DocumentClientException de) { if (!ct.IsCancellationRequested) { messages.Enqueue(new ColoredMessage($"Cosmos DB request {thisRequest} eventually failed with: {de.Message}; Retry-after: {de.RetryAfter.TotalSeconds} seconds.", Color.Red)); } _cosmosRequestsFailed++; } catch (Exception e) { if (!ct.IsCancellationRequested) { messages.Enqueue(new ColoredMessage($"Cosmos DB request {thisRequest} eventually failed with: {e.Message}", Color.Red)); } _cosmosRequestsFailed++; } }, combinedToken) .ContinueWith((t, k) => { if (t.IsFaulted) { messages.Enqueue(new ColoredMessage($"Request to Cosmos DB failed with: {t.Exception?.Flatten().InnerExceptions.First().Message}", Color.Red)); } _cosmosRequestsFailed++; }, thisRequest, TaskContinuationOptions.NotOnRanToCompletion) ); #endregion Write to Cosmos DB if (_totalMessages % 500 == 0) { eventHubsTimer.Stop(); cosmosTimer.Stop(); _cosmosTotalElapsedTime += _cosmosElapsedTime; _cosmosRequestsSucceeded += _cosmosRequestsSucceededInBatch; // Calculate RUs/second/month: var ruPerSecond = (_cosmosRUsPerBatch / (_cosmosElapsedTime * .001)); var ruPerMonth = ruPerSecond * 86400 * 30; // Random delay every 1000 messages that are sent. //await Task.Delay(random.Next(100, 1000), externalCancellationToken).ConfigureAwait(false); await Task.Delay(5000, externalCancellationToken); // The obvious and recommended method for sending a lot of data is to do so in batches. This method can // multiply the amount of data sent with each request by hundreds or thousands. However, the point of // our exercise is not to maximize throughput and send as much data as possible, but to compare the // relative performance between Event Hubs and Cosmos DB. // Output statistics. Be on the lookout for the following: // - Inserted line shows successful inserts in this batch and throughput for writes/second with RU/s usage and estimated monthly ingestion rate added to Cosmos DB statistics. // - Processing time: Processing time for the past 250 requested inserts. // - Total elapsed time: Running total of time taken to process all documents. // - Succeeded shows number of accumulative successful inserts to the service. // - Pending are items in the bulkhead queue. This amount will continue to grow if the service is unable to keep up with demand. // - Accumulative failed requests that encountered an exception. messages.Enqueue(new ColoredMessage($"Total requests: requested {_totalMessages:00} ", Color.Cyan)); messages.Enqueue(new ColoredMessage(string.Empty)); messages.Enqueue(new ColoredMessage($"Inserted {_cosmosRequestsSucceededInBatch:00} docs @ {(_cosmosRequestsSucceededInBatch / (_cosmosElapsedTime * .001)):0.00} writes/s, {ruPerSecond:0.00} RU/s ({(ruPerMonth / (1000 * 1000 * 1000)):0.00}B max monthly 1KB writes) ", Color.White)); messages.Enqueue(new ColoredMessage($"Processing time {_cosmosElapsedTime} ms", Color.Magenta)); messages.Enqueue(new ColoredMessage($"Total elapsed time {(_cosmosTotalElapsedTime * .001):0.00} seconds", Color.Magenta)); messages.Enqueue(new ColoredMessage($"Total succeeded {_cosmosRequestsSucceeded:00} ", Color.Green)); messages.Enqueue(new ColoredMessage($"Total pending {_cosmosRequestsMade - _cosmosRequestsSucceeded - _cosmosRequestsFailed:00} ", Color.Yellow)); messages.Enqueue(new ColoredMessage($"Total failed {_cosmosRequestsFailed:00}", Color.Red)); messages.Enqueue(new ColoredMessage(string.Empty)); // Restart timers and reset batch settings: cosmosTimer.Restart(); _cosmosElapsedTime = 0; _cosmosRUsPerBatch = 0; _cosmosRequestsSucceededInBatch = 0; // Output all messages available right now, in one go. progress.Report(ProgressWithMessages(ConsumeAsEnumerable(messages))); } //await Task.Delay(random.Next(200, 500), externalCancellationToken).ConfigureAwait(false); } messages.Enqueue(new ColoredMessage("Data generation complete", Color.Magenta)); progress.Report(ProgressWithMessages(ConsumeAsEnumerable(messages))); BulkheadForCosmosDbCalls.Dispose(); eventHubsTimer.Stop(); cosmosTimer.Stop(); }
// Send data to Event Hub: private static async Task SendData(int randomSeed, EventHubClient eventHubClient, int waittime, CancellationToken externalCancellationToken, IProgress <Progress> progress) { if (waittime > 0) { var span = TimeSpan.FromMilliseconds(waittime); await Task.Delay(span, externalCancellationToken); } if (externalCancellationToken == null) { throw new ArgumentNullException(nameof(externalCancellationToken)); } if (progress == null) { throw new ArgumentNullException(nameof(progress)); } // Perform garbage collection prior to timing for statistics. GC.Collect(); GC.WaitForPendingFinalizers(); var internalCancellationTokenSource = new CancellationTokenSource(); var combinedToken = CancellationTokenSource.CreateLinkedTokenSource(externalCancellationToken, internalCancellationTokenSource.Token).Token; var random = new Random(randomSeed); var tasks = new List <Task>(); var messages = new ConcurrentQueue <ColoredMessage>(); var eventHubsTimer = new Stopwatch(); // Ensure none of what follows runs synchronously. await Task.FromResult(true).ConfigureAwait(false); // Continue while cancellation is not requested. while (!combinedToken.IsCancellationRequested) { if (externalCancellationToken.IsCancellationRequested) { return; } _totalMessages++; var thisRequest = _totalMessages; var carEvent = TelemetryGenerator.GenerateMessage(); #region Write to Event Hub _eventHubRequestsMade++; tasks.Add(BulkheadForEventHubCalls.ExecuteAsync(async ct => { try { var eventData = new EventData(Encoding.UTF8.GetBytes(carEvent.GetData())); eventHubsTimer.Start(); // Send to Event Hub: await eventHubClient.SendAsync(eventData: eventData, partitionKey: carEvent.region).ConfigureAwait(false); eventHubsTimer.Stop(); _eventHubElapsedTime = eventHubsTimer.ElapsedMilliseconds; // Increment the count of number of Event Hub requests that succeeded. _eventHubRequestsSucceededInBatch++; } catch (Exception e) { if (!ct.IsCancellationRequested) { messages.Enqueue(new ColoredMessage($"Event Hubs request {thisRequest} eventually failed with: {e.Message}", Color.Red)); } _eventHubRequestsFailed++; } }, combinedToken) .ContinueWith((t, k) => { if (t.IsFaulted) { messages.Enqueue(new ColoredMessage($"Request to Event Hubs failed with: {t.Exception?.Flatten().InnerExceptions.First().Message}", Color.Red)); } _eventHubRequestsFailed++; }, thisRequest, TaskContinuationOptions.NotOnRanToCompletion) ); #endregion Write to Event Hub if (_totalMessages % 500 == 0) { eventHubsTimer.Stop(); _eventHubTotalElapsedTime += _eventHubElapsedTime; _eventHubRequestsSucceeded += _eventHubRequestsSucceededInBatch; // Random delay every 500 messages that are sent. //await Task.Delay(random.Next(100, 1000), externalCancellationToken).ConfigureAwait(false); await Task.Delay(random.Next(1000, 3000), externalCancellationToken); // The obvious and recommended method for sending a lot of data is to do so in batches. This method can // multiply the amount of data sent with each request by hundreds or thousands. // Output statistics. Be on the lookout for the following: // - Inserted line shows successful inserts in this batch and throughput for writes/second. // - Processing time: Processing time for the past 250 requested inserts. // - Total elapsed time: Running total of time taken to process all documents. // - Succeeded shows number of accumulative successful inserts to the service. // - Pending are items in the bulkhead queue. This amount will continue to grow if the service is unable to keep up with demand. // - Accumulative failed requests that encountered an exception. messages.Enqueue(new ColoredMessage(string.Empty)); messages.Enqueue(new ColoredMessage($"Event Hub: sent {_eventHubRequestsSucceededInBatch:00} events this batch @ {(_eventHubRequestsSucceededInBatch / (_eventHubElapsedTime * .001)):0.00} writes/s ", Color.White)); messages.Enqueue(new ColoredMessage($"Event Hub: total events sent {_eventHubRequestsMade:00} ", Color.Green)); messages.Enqueue(new ColoredMessage($"Event Hub: processing time {_eventHubElapsedTime} ms", Color.Magenta)); messages.Enqueue(new ColoredMessage($"Event Hub: total elapsed time {(_eventHubTotalElapsedTime * .001):0.00} seconds", Color.Magenta)); //messages.Enqueue(new ColoredMessage($"Event Hub: total succeeded {_eventHubRequestsSucceeded:00} ", Color.Green)); //messages.Enqueue(new ColoredMessage($"Event Hub: total pending {_eventHubRequestsMade - _eventHubRequestsSucceeded - _eventHubRequestsFailed:00} ", Color.Yellow)); messages.Enqueue(new ColoredMessage($"Event Hub: total failed {_eventHubRequestsFailed:00}", Color.Red)); eventHubsTimer.Restart(); _eventHubElapsedTime = 0; _eventHubRequestsSucceededInBatch = 0; // Output all messages available right now, in one go. progress.Report(ProgressWithMessages(ConsumeAsEnumerable(messages))); } // Add short delay to prevent pumping too many events too fast. await Task.Delay(random.Next(5, 15), externalCancellationToken).ConfigureAwait(false); } messages.Enqueue(new ColoredMessage("Data generation complete", Color.Magenta)); progress.Report(ProgressWithMessages(ConsumeAsEnumerable(messages))); BulkheadForEventHubCalls.Dispose(); eventHubsTimer.Stop(); }
public static void Main(string[] args) { // Setup configuration to either read from the appsettings.json file (if present) or environment variables. var builder = new ConfigurationBuilder() .SetBasePath(Directory.GetCurrentDirectory()) .AddJsonFile("appsettings.json", optional: true, reloadOnChange: true) .AddEnvironmentVariables(); _configuration = builder.Build(); var arguments = ParseArguments(); var cosmosDbConnectionString = new CosmosDbConnectionString(arguments.CosmosDbConnectionString); // Set an optional timeout for the generator. var cancellationSource = arguments.MillisecondsToRun == 0 ? new CancellationTokenSource() : new CancellationTokenSource(arguments.MillisecondsToRun); var cancellationToken = cancellationSource.Token; var statistics = new Statistic[0]; // Set the Cosmos DB connection policy. var connectionPolicy = new ConnectionPolicy { ConnectionMode = ConnectionMode.Direct, ConnectionProtocol = Protocol.Tcp, UseMultipleWriteLocations = true }; var numberOfMillisecondsToLead = arguments.MillisecondsToLead; var taskWaitTime = 0; if (numberOfMillisecondsToLead > 0) { taskWaitTime = numberOfMillisecondsToLead; } var progress = new Progress <Progress>(); progress.ProgressChanged += (sender, progressArgs) => { foreach (var message in progressArgs.Messages) { WriteLineInColor(message.Message, message.Color.ToConsoleColor()); } statistics = progressArgs.Statistics; }; WriteLineInColor("Vehicle Telemetry Generator", ConsoleColor.White); Console.WriteLine("======"); WriteLineInColor("Press Ctrl+C or Ctrl+Break to cancel.", ConsoleColor.Cyan); Console.WriteLine("Statistics for generated vehicle telemetry data will be updated for every 500 sent"); Console.WriteLine(string.Empty); ThreadPool.SetMinThreads(100, 100); // Handle Control+C or Control+Break. Console.CancelKeyPress += (o, e) => { WriteLineInColor("Stopped generator. No more events are being sent.", ConsoleColor.Yellow); cancellationSource.Cancel(); // Allow the main thread to continue and exit... WaitHandle.Set(); OutputStatistics(statistics); }; // Initialize the telemetry generator: TelemetryGenerator.Init(); // Instantiate Cosmos DB client and start sending messages: using (_cosmosDbClient = new DocumentClient(cosmosDbConnectionString.ServiceEndpoint, cosmosDbConnectionString.AuthKey, connectionPolicy)) { InitializeCosmosDb().Wait(); // Find and output the collection details, including # of RU/s. var dataCollection = GetCollectionIfExists(DatabaseName, CollectionName); var offer = (OfferV2)_cosmosDbClient.CreateOfferQuery().Where(o => o.ResourceLink == dataCollection.SelfLink).AsEnumerable().FirstOrDefault(); if (offer != null) { var currentCollectionThroughput = offer.Content.OfferThroughput; WriteLineInColor($"Found collection `{CollectionName}` with {currentCollectionThroughput} RU/s ({currentCollectionThroughput} reads/second; {currentCollectionThroughput / 5} writes/second @ 1KB doc size)", ConsoleColor.Green); var estimatedCostPerMonth = 0.06 * offer.Content.OfferThroughput; var estimatedCostPerHour = estimatedCostPerMonth / (24 * 30); WriteLineInColor($"The collection will cost an estimated ${estimatedCostPerHour:0.00} per hour (${estimatedCostPerMonth:0.00} per month (per write region))", ConsoleColor.Green); } // Start sending data to Cosmos DB. SendData(100, taskWaitTime, cancellationToken, progress).Wait(); } cancellationSource.Cancel(); Console.WriteLine(); WriteLineInColor("Done sending generated vehicle telemetry data", ConsoleColor.Cyan); Console.WriteLine(); Console.WriteLine(); OutputStatistics(statistics); // Keep the console open. Console.ReadLine(); WaitHandle.WaitOne(); }
public static void Main(string[] args) { // Setup configuration to either read from the appsettings.json file (if present) or environment variables. var builder = new ConfigurationBuilder() .SetBasePath(Directory.GetCurrentDirectory()) .AddJsonFile("appsettings.json", optional: true, reloadOnChange: true) .AddEnvironmentVariables(); _configuration = builder.Build(); var arguments = ParseArguments(); // Set an optional timeout for the generator. var cancellationSource = arguments.MillisecondsToRun == 0 ? new CancellationTokenSource() : new CancellationTokenSource(arguments.MillisecondsToRun); var cancellationToken = cancellationSource.Token; var statistics = new Statistic[0]; var numberOfMillisecondsToLead = arguments.MillisecondsToLead; var taskWaitTime = 0; if (numberOfMillisecondsToLead > 0) { taskWaitTime = numberOfMillisecondsToLead; } var progress = new Progress <Progress>(); progress.ProgressChanged += (sender, progressArgs) => { foreach (var message in progressArgs.Messages) { WriteLineInColor(message.Message, message.Color.ToConsoleColor()); } statistics = progressArgs.Statistics; }; WriteLineInColor("Vehicle Telemetry Generator", ConsoleColor.White); Console.WriteLine("======"); WriteLineInColor("Press Ctrl+C or Ctrl+Break to cancel.", ConsoleColor.Cyan); Console.WriteLine("Statistics for generated vehicle telemetry data will be updated for every 500 sent"); Console.WriteLine(string.Empty); ThreadPool.SetMinThreads(100, 100); // Handle Control+C or Control+Break. Console.CancelKeyPress += (o, e) => { WriteLineInColor("Stopped generator. No more events are being sent.", ConsoleColor.Yellow); cancellationSource.Cancel(); // Allow the main thread to continue and exit... WaitHandle.Set(); OutputStatistics(statistics); }; // Initialize the telemetry generator: TelemetryGenerator.Init(); // Create an Event Hub Client from a connection string, using the EventHubConnectionString value. var eventHubClient = EventHubClient.CreateFromConnectionString( arguments.EventHubConnectionString ); // Start sending data to Event Hub. SendData(100, eventHubClient, taskWaitTime, cancellationToken, progress).Wait(); cancellationSource.Cancel(); Console.WriteLine(); WriteLineInColor("Done sending generated vehicle telemetry data", ConsoleColor.Cyan); Console.WriteLine(); Console.WriteLine(); OutputStatistics(statistics); // Keep the console open. Console.ReadLine(); WaitHandle.WaitOne(); }