static async Task HandleChangesAsync(IReadOnlyCollection <object> changes, CancellationToken cancellationToken) { MongoSink.BlobUploader blobUploader = new MongoSink.BlobUploader( blobConnectionString, blobContainer); MongoSink.MongoSink mongoSink = new MongoSink.MongoSink(destConnectionString, destDBName, destContainerName, insertRetries, blobConnectionString, blobContainer, isUpsert); SourceTransformation sourceTransformation = null; if (!string.IsNullOrEmpty(sourceKeyTransformationString)) { try { sourceTransformation = JsonConvert.DeserializeObject <SourceTransformation> (sourceKeyTransformationString); } catch (Exception exp) { Console.WriteLine(exp.Message); } } Transformation transformation = new Transformation(transformationType, sourceKeys, destinationKey, delimiter, sourceTransformation, swapKeyValues); Console.WriteLine("Received data from the source:" + changes.Count.ToString()); foreach (object item in changes) { try { //Console.WriteLine(item.ToString()); JObject objects = JObject.Parse(item.ToString()); objects.Remove("_lsn"); objects.Remove("_metadata"); objects.Remove("_id"); string json = objects.ToString(); MongoDB.Bson.BsonDocument bdoc = CosmosDbSchemaDecoder.GetBsonDocument(json, false); if (transformationType != "NONE") { transformation.Execute(bdoc); } await mongoSink.InsertAsync(bdoc, source.Token); } catch (Exception exp) { blobUploader.Upload(item.ToString()); } } }
private async Task InsertDocument(dynamic doc) { bool isSucceed = false; try { JObject objects = JObject.Parse(doc.ToString()); objects.Remove("_lsn"); objects.Remove("_metadata"); BsonDocument bsonDocument = null; // Old schema is plain json, no need to convert it. if (schema.ToLower() == "old") { bsonDocument = BsonSerializer.Deserialize <BsonDocument>(objects.ToString()); } else { bsonDocument = CosmosDbSchemaDecoder.GetBsonDocument(objects.ToString(), false); } for (int i = 0; i < insertRetries; i++) { try { await destDocStoreCollection.InsertOneAsync(bsonDocument); isSucceed = true; //Operation succeed just break the loop break; } catch (Exception ex) { if (!IsThrottled(ex)) { this.logger.LogError("ERROR: With collection {0}", ex.ToString()); throw; } else { // Thread will wait in between 1.5 secs and 3 secs. await Task.Delay(rnd.Next(minWait, maxWait)); } } } if (!isSucceed) { insertFailedDocs.Add(objects.ToString()); } } catch (Exception e) { // Parse Error parseErrors.Add(doc.ToString()); } }