async Task ProcessChangesAsync(IReadOnlyCollection <Document> docs, CancellationToken cancellationToken) { Boolean isSyntheticKey = SourcePartitionKeys.Contains(","); Boolean isNestedAttribute = SourcePartitionKeys.Contains("/"); Container targetContainer = destinationCollectionClient.GetContainer(config.DestDbName, config.DestCollectionName); containerToStoreDocuments = targetContainer; Document document; BulkOperations <Document> bulkOperations = new BulkOperations <Document>(docs.Count); foreach (Document doc in docs) { //Console.WriteLine($"\tDetected operation..."); document = (SourcePartitionKeys != null & (!(SourcePartitionKeys.Equals("")))) ? MapPartitionKey(doc, isSyntheticKey, TargetPartitionKey, isNestedAttribute, SourcePartitionKeys) : document = doc; bulkOperations.Tasks.Add(containerToStoreDocuments.UpsertItemAsync(item: document, cancellationToken: cancellationToken).CaptureOperationResponse(document)); } BulkOperationResponse <Document> bulkOperationResponse = await bulkOperations.ExecuteAsync(); if (bulkOperationResponse.Failures.Count > 0 && containerClient != null) { Console.WriteLine($"\tFailed docs!"); WriteFailedDocsToBlob("FailedImportDocs", containerClient, bulkOperationResponse); } LogMetrics(bulkOperationResponse); }
private static void LogMetrics(BulkOperationResponse <Document> bulkOperationResponse) { Program.telemetryClient.TrackMetric("TotalInserted", bulkOperationResponse.SuccessfulDocuments); Program.telemetryClient.TrackMetric("InsertedDocuments-Process:" + Process.GetCurrentProcess().Id, bulkOperationResponse.SuccessfulDocuments); Program.telemetryClient.TrackMetric("TotalRUs", bulkOperationResponse.TotalRequestUnitsConsumed); if (bulkOperationResponse.Failures.Count > 0) { Program.telemetryClient.TrackMetric("BadInputDocsCount", bulkOperationResponse.Failures.Count); } if (bulkOperationResponse.Failures.Count > 0) { Program.telemetryClient.TrackMetric("FailedImportDocsCount", bulkOperationResponse.Failures.Count); } }
private static void WriteFailedDocsToBlob(string failureType, BlobContainerClient containerClient, BulkOperationResponse <Document> bulkOperationResponse) { string failedDocs; string failures; byte[] byteArray; BlobClient blobClient = containerClient.GetBlobClient(failureType + Guid.NewGuid().ToString() + ".csv"); failures = JsonConvert.SerializeObject(String.Join(",", bulkOperationResponse.DocFailures)); failedDocs = JsonConvert.SerializeObject(String.Join(",", bulkOperationResponse.FailedDocs)); failedDocs = Regex.Replace(failedDocs, @"\\r\\n?|\\n?|\\\?|\\", ""); byteArray = Encoding.ASCII.GetBytes(failures + ", " + bulkOperationResponse.Failures.Count + "|" + failedDocs); using (var ms = new MemoryStream(byteArray)) { blobClient.Upload(ms, overwrite: true); } }