public async Task <bool> CreateUploadTestDataInbatches()
{
#region batchVariables
//initialize Batch Process variables
int batchCount = 0;
int totalUploaded = 0;
var badEntities = new List <Object>();
Stopwatch stopwatch = new Stopwatch();
stopwatch.Start();
#endregion
while (totalUploaded < maxtestDocumentCount)
{
batchCount++;
logger.LogInfo("Begin Sample Db creation");
List <dynamic> entities = GetCommonEntitiesinBatch();
BulkImportResponse uploadResponse = new BulkImportResponse();
if (entities.Any())
{
uploadResponse = await cosmosBulkImporter.BulkSendToNewCollection <dynamic>(entities);
}
badEntities = uploadResponse.BadInputDocuments;
//summary.totalRecordsSent += uploadResponse.NumberOfDocumentsImported;
totalUploaded += entities.Count();
logger.LogInfo($"Summary of Batch {batchCount} records retrieved {entities.Count()}. Records Uploaded: {uploadResponse.NumberOfDocumentsImported}");
//logger.LogInfo($"Total records retrieved {summary.totalRecordsRetrieved}. Total records uploaded {summary.totalRecordsSent}");
logger.LogInfo($"Time elapsed : {stopwatch.Elapsed} ");
}
stopwatch.Stop();
logger.LogInfo("Completed Sample DB creation.");
return(true);
}
public async Task ReadUploadInBatches(IDocumentQuery <string> query, List <ScrubRule> scrubRules)
{
#region batchVariables
//initialize Batch Process variables
int batchCount = 0;
TotalRecordsRetrieved = 0;
TotalRecordsScrubbed = 0;
var badEntities = new List <Object>();
Stopwatch stopwatch = new Stopwatch();
stopwatch.Start();
var objScrubber = new ObjectScrubber();
#endregion
while (query.HasMoreResults)
{
batchCount++;
logger.LogInfo($"BatchNumber : {batchCount} begins ");
List <string> entities = await GetCommonStringEntitiesinBatch(query);
TotalRecordsRetrieved += entities.Count();
BulkImportResponse uploadResponse = new BulkImportResponse();
var scrubbedEntities = entities;
if (entities.Any())
{
var jEntities = new List <JToken>();
foreach (var scrubRule in scrubRules)
{
jEntities = objScrubber.ScrubObjectList(scrubbedEntities, scrubRule);
var nentities = new List <string>();
foreach (var jobj in jEntities)
{
nentities.Add(JsonConvert.SerializeObject(jobj));
}
scrubbedEntities = nentities;
scrubRule.RecordsUpdated += jEntities.Count;
}
var objEntities = jEntities.Cast <Object>().ToList();
try
{
uploadResponse = await cosmosBulkImporter.BulkSendToNewCollection <dynamic>(objEntities);
}
catch (Exception ex)
{
logger.LogError(ex);
throw;
}
}
badEntities = uploadResponse.BadInputDocuments;
TotalRecordsScrubbed += uploadResponse.NumberOfDocumentsImported;
logger.LogInfo($"Summary of Batch {batchCount} records retrieved {entities.Count()}. Records Uploaded: {uploadResponse.NumberOfDocumentsImported}");
logger.LogInfo($"Total records retrieved {TotalRecordsRetrieved}. Total records uploaded {TotalRecordsScrubbed}");
logger.LogInfo($"Time elapsed : {stopwatch.Elapsed} ");
}
stopwatch.Stop();
logger.LogInfo("Document Scrubbing completed");
}
public async Task ReadUploadInBatches(IDocumentQuery <dynamic> query)
{
#region batchVariables
//initialize Batch Process variables
int batchCount = 0;
TotalRecordsRetrieved = 0;
TotalRecordsSent = 0;
var badEntities = new List <Object>();
Stopwatch stopwatch = new Stopwatch();
stopwatch.Start();
#endregion
while (query.HasMoreResults)
{
batchCount++;
logger.LogInfo($"BatchNumber : {batchCount} begins ");
List <dynamic> entities = await GetCommonEntitiesinBatch(query);
TotalRecordsRetrieved += entities.Count();
List <object> objEntities = new List <object>();
objEntities.AddRange((IEnumerable <object>)entities);
List <string> strEntities = new List <string>();
foreach (var obj in objEntities)
{
strEntities.Add(JsonConvert.SerializeObject(obj));
}
BulkImportResponse uploadResponse = new BulkImportResponse();
var scrubbedEntities = strEntities;
if (entities.Any())
{
if (noFilterScrubRules == null || noFilterScrubRules.Count == 0)
{
uploadResponse = await cosmosBulkImporter.BulkSendToNewCollection <dynamic>(entities);
}
else
{
var jEntities = new List <JToken>();
foreach (var sRule in noFilterScrubRules)
{
jEntities = objectScrubber.ScrubObjectList(scrubbedEntities, sRule);
var nentities = new List <string>();
foreach (var jobj in jEntities)
{
nentities.Add(JsonConvert.SerializeObject(jobj));
}
scrubbedEntities = nentities;
sRule.RecordsUpdated += jEntities.Count;
}
var objDocuments = jEntities.Cast <Object>().ToList();
uploadResponse = await cosmosBulkImporter.BulkSendToNewCollection <dynamic>(objDocuments);
}
}
else
{
logger.LogInfo("No Entities retrieved from query");
continue;
}
badEntities = uploadResponse.BadInputDocuments;
TotalRecordsSent += uploadResponse.NumberOfDocumentsImported;
logger.LogInfo($"Summary of Batch {batchCount} records retrieved {entities.Count()}. Records Uploaded: {uploadResponse.NumberOfDocumentsImported}");
logger.LogInfo($"Total records retrieved {TotalRecordsRetrieved}. Total records uploaded {TotalRecordsSent}");
logger.LogInfo($"Time elapsed : {stopwatch.Elapsed} ");
}
SetCompleteOnNoFilterRules();
stopwatch.Stop();
logger.LogInfo("Document Migration completed");
}
