async Task ProcessChangesAsync(IReadOnlyCollection <Document> docs, CancellationToken cancellationToken)
{
Boolean isSyntheticKey = SourcePartitionKeys.Contains(",");
Boolean isNestedAttribute = SourcePartitionKeys.Contains("/");
Container targetContainer = destinationCollectionClient.GetContainer(config.DestDbName, config.DestCollectionName);
containerToStoreDocuments = targetContainer;
Document document;
BulkOperations <Document> bulkOperations = new BulkOperations <Document>(docs.Count);
foreach (Document doc in docs)
{
//Console.WriteLine($"\tDetected operation...");
document = (SourcePartitionKeys != null & (!(SourcePartitionKeys.Equals("")))) ? MapPartitionKey(doc, isSyntheticKey, TargetPartitionKey, isNestedAttribute, SourcePartitionKeys) : document = doc;
bulkOperations.Tasks.Add(containerToStoreDocuments.UpsertItemAsync(item: document, cancellationToken: cancellationToken).CaptureOperationResponse(document));
}
BulkOperationResponse <Document> bulkOperationResponse = await bulkOperations.ExecuteAsync();
if (bulkOperationResponse.Failures.Count > 0 && containerClient != null)
{
Console.WriteLine($"\tFailed docs!");
WriteFailedDocsToBlob("FailedImportDocs", containerClient, bulkOperationResponse);
}
LogMetrics(bulkOperationResponse);
}
private static void LogMetrics(BulkOperationResponse <Document> bulkOperationResponse)
{
Program.telemetryClient.TrackMetric("TotalInserted", bulkOperationResponse.SuccessfulDocuments);
Program.telemetryClient.TrackMetric("InsertedDocuments-Process:"
+ Process.GetCurrentProcess().Id, bulkOperationResponse.SuccessfulDocuments);
Program.telemetryClient.TrackMetric("TotalRUs", bulkOperationResponse.TotalRequestUnitsConsumed);
if (bulkOperationResponse.Failures.Count > 0)
{
Program.telemetryClient.TrackMetric("BadInputDocsCount", bulkOperationResponse.Failures.Count);
}
if (bulkOperationResponse.Failures.Count > 0)
{
Program.telemetryClient.TrackMetric("FailedImportDocsCount", bulkOperationResponse.Failures.Count);
}
}
private static void WriteFailedDocsToBlob(string failureType, BlobContainerClient containerClient, BulkOperationResponse <Document> bulkOperationResponse)
{
string failedDocs;
string failures;
byte[] byteArray;
BlobClient blobClient = containerClient.GetBlobClient(failureType + Guid.NewGuid().ToString() + ".csv");
failures = JsonConvert.SerializeObject(String.Join(",", bulkOperationResponse.DocFailures));
failedDocs = JsonConvert.SerializeObject(String.Join(",", bulkOperationResponse.FailedDocs));
failedDocs = Regex.Replace(failedDocs, @"\\r\\n?|\\n?|\\\?|\\", "");
byteArray = Encoding.ASCII.GetBytes(failures + ", " + bulkOperationResponse.Failures.Count + "|" + failedDocs);
using (var ms = new MemoryStream(byteArray))
{
blobClient.Upload(ms, overwrite: true);
}
}
