static async Task HandleChangesAsync(IReadOnlyCollection <object> changes, CancellationToken cancellationToken)
{
MongoSink.BlobUploader blobUploader = new MongoSink.BlobUploader(
blobConnectionString,
blobContainer);
MongoSink.MongoSink mongoSink = new MongoSink.MongoSink(destConnectionString,
destDBName,
destContainerName,
insertRetries,
blobConnectionString,
blobContainer,
isUpsert);
SourceTransformation sourceTransformation = null;
if (!string.IsNullOrEmpty(sourceKeyTransformationString))
{
try
{
sourceTransformation = JsonConvert.DeserializeObject <SourceTransformation>
(sourceKeyTransformationString);
}
catch (Exception exp)
{
Console.WriteLine(exp.Message);
}
}
Transformation transformation = new Transformation(transformationType,
sourceKeys,
destinationKey,
delimiter,
sourceTransformation,
swapKeyValues);
Console.WriteLine("Received data from the source:" + changes.Count.ToString());
foreach (object item in changes)
{
try
{
//Console.WriteLine(item.ToString());
JObject objects = JObject.Parse(item.ToString());
objects.Remove("_lsn");
objects.Remove("_metadata");
objects.Remove("_id");
string json = objects.ToString();
MongoDB.Bson.BsonDocument bdoc = CosmosDbSchemaDecoder.GetBsonDocument(json, false);
if (transformationType != "NONE")
{
transformation.Execute(bdoc);
}
await mongoSink.InsertAsync(bdoc, source.Token);
}
catch (Exception exp)
{
blobUploader.Upload(item.ToString());
}
}
}
private async Task InsertDocument(dynamic doc)
{
bool isSucceed = false;
try
{
JObject objects = JObject.Parse(doc.ToString());
objects.Remove("_lsn");
objects.Remove("_metadata");
BsonDocument bsonDocument = null;
// Old schema is plain json, no need to convert it.
if (schema.ToLower() == "old")
{
bsonDocument = BsonSerializer.Deserialize <BsonDocument>(objects.ToString());
}
else
{
bsonDocument = CosmosDbSchemaDecoder.GetBsonDocument(objects.ToString(), false);
}
for (int i = 0; i < insertRetries; i++)
{
try
{
await destDocStoreCollection.InsertOneAsync(bsonDocument);
isSucceed = true;
//Operation succeed just break the loop
break;
}
catch (Exception ex)
{
if (!IsThrottled(ex))
{
this.logger.LogError("ERROR: With collection {0}", ex.ToString());
throw;
}
else
{
// Thread will wait in between 1.5 secs and 3 secs.
await Task.Delay(rnd.Next(minWait, maxWait));
}
}
}
if (!isSucceed)
{
insertFailedDocs.Add(objects.ToString());
}
}
catch (Exception e)
{
// Parse Error
parseErrors.Add(doc.ToString());
}
}
