public static async Task RunAsync([EventHubTrigger("%EventHubName%", Connection = "EventHubsConnectionString", ConsumerGroup = "%ConsumerGroup%")] EventData[] eventHubData, ILogger log) { var client = await CosmosDBClient.GetClient(); var tasks = new List <Task <ResourceResponse <Document> > >(); Stopwatch sw = new Stopwatch(); sw.Start(); double totalRUbyBatch = 0; int positionInBatch = 1; foreach (var data in eventHubData) { try { string message = Encoding.UTF8.GetString(data.Body.Array); var documentPayload = new { eventData = JObject.Parse(message), enqueuedAt = data.SystemProperties.EnqueuedTimeUtc, storedAt = DateTime.UtcNow, positionInBatch }; tasks.Add(client.CreateDocumentAsync(documentPayload)); positionInBatch += 1; } catch (Exception ex) { log.LogError($"{ex} - {ex.Message}"); } } await Task.WhenAll(tasks); foreach (var t in tasks) { totalRUbyBatch += t.GetAwaiter().GetResult().RequestCharge; } sw.Stop(); string logMessage = $"[Test1] T:{eventHubData.Length} doc - E:{sw.ElapsedMilliseconds} msec"; if (eventHubData.Length > 0) { logMessage += Environment.NewLine + $"AVG:{(sw.ElapsedMilliseconds / eventHubData.Length):N3} msec"; logMessage += Environment.NewLine + $"RU:{totalRUbyBatch}. AVG RU:{(totalRUbyBatch / eventHubData.Length):N3}"; } log.LogInformation(logMessage); }
public static async Task <CosmosDBClient> GetClient() { await semaphoreSlim.WaitAsync(); try { return(_instance ?? (_instance = await CreateInstance())); } finally { semaphoreSlim.Release(); } }
public static async Task RunAsync( ExecutionContext ctx, [KafkaTrigger("%KafkaBrokers%", "%EventHubName%", ConsumerGroup = "%ConsumerGroup%", EventHubConnectionString = "EventHubsConnectionString", Protocol = BrokerProtocol.SaslSsl, AuthenticationMode = BrokerAuthenticationMode.Plain, SslCaLocation = "D:\\home\\site\\wwwroot\\cacert.pem", // TODO: come with a better way to specify the path in KafkaTrigger Username = "******", Password = "******")] KafkaEventData <string>[] kafkaEvents, [CosmosDB(databaseName: "%CosmosDBDatabaseName%", collectionName: "%CosmosDBCollectionName%", ConnectionStringSetting = "CosmosDBConnectionString")] IAsyncCollector <JObject> cosmosMessage, ILogger log) { var client = await CosmosDBClient.GetClient(); var tasks = new List <Task <ResourceResponse <Document> > >(); long len = 0; Stopwatch sw = new Stopwatch(); sw.Start(); double totalRUbyBatch = 0; int positionInBatch = 1; foreach (var data in kafkaEvents) { try { string message = data.Value; len += message.Length; var document = JObject.Parse(message); document["id"] = document["eventId"]; document["enqueuedAt"] = data.Timestamp; document["processedAt"] = DateTime.UtcNow; document["positionInBatch"] = positionInBatch; tasks.Add(client.CreateDocumentAsync(document)); positionInBatch += 1; } catch (Exception ex) { log.LogError($"{ex} - {ex.Message}"); } } await Task.WhenAll(tasks); foreach (var t in tasks) { totalRUbyBatch += t.GetAwaiter().GetResult().RequestCharge; } sw.Stop(); string logMessage = $"[Test1] T:{len} doc - E:{sw.ElapsedMilliseconds} msec"; if (len > 0) { logMessage += Environment.NewLine + $"AVG:{(sw.ElapsedMilliseconds / len):N3} msec"; logMessage += Environment.NewLine + $"RU:{totalRUbyBatch}. AVG RU:{(totalRUbyBatch / len):N3}"; } log.LogInformation(logMessage); }