private static async Task ImportVerticesAsync(IEnumerable <GremlinVertex> gremlinVertices)
{
Trace.TraceInformation(nameof(ImportVerticesAsync));
var token = new CancellationTokenSource().Token;
BulkImportResponse vResponse = null;
while (Client == null || _graphBulkExecutor == null || _initializeAsyncTask?.IsCompleted != true)
{
await Task.Delay(100.Milliseconds(), token);
}
try
{
if (gremlinVertices != null)
{
vResponse = await _graphBulkExecutor.BulkImportAsync(
gremlinVertices,
true,
true,
null,
null,
token);
}
}
catch (DocumentClientException de)
{
Trace.TraceError("Document client exception: {0}", de);
}
catch (Exception e)
{
Trace.TraceError("Exception: {0}", e);
}
Trace.TraceInformation("END" + nameof(ImportVerticesAsync));
}
public async Task BulkInsertVertices(IEnumerable <V> vertices, string partition, CancellationToken cancel)
{
await InitBulkExecutor();
var objs = vertices.Select(ToVertex).ToList();
logger.LogInformation($"Adding vertices to partition {partition}...{objs.Count}");
int pos = 0;
int totalInserted = 0;
while (pos < objs.Count)
{
var batch = objs.Skip(pos).Take(100);
var retryCount = 0;
var succeed = false;
while (!succeed && retryCount < 3)
{
try
{
await bulkExecutor.BulkImportAsync(batch, true, true, null, null, cancel);
succeed = true;
}
catch (DocumentClientException ex) when(ex.StatusCode == (HttpStatusCode)429)
{
retryCount++;
Thread.Sleep(TimeSpan.FromSeconds(1));
}
}
totalInserted += batch.Count();
logger.LogInformation($"executing batch...{totalInserted}");
pos += batch.Count();
}
}
public async Task ProcessChangesAsync(
IChangeFeedObserverContext context,
IReadOnlyList <Document> docs,
CancellationToken cancellationToken)
{
try
{
List <Document> transformedDocs = new List <Document>();
foreach (var doc in docs)
{
transformedDocs.AddRange(documentTransformer.TransformDocument(doc).Result);
}
BulkImportResponse bulkImportResponse = await bulkExecutor.BulkImportAsync(
documents : transformedDocs,
enableUpsert : true,
maxConcurrencyPerPartitionKeyRange : 1,
disableAutomaticIdGeneration : true,
maxInMemorySortingBatchSize : null,
cancellationToken : new CancellationToken());
LogMetrics(context, bulkImportResponse);
}
catch (Exception e)
{
Program.telemetryClient.TrackException(e);
}
Program.telemetryClient.Flush();
}
public async Task <int> BulkImport(IEnumerable <T> entities, CancellationToken token = new CancellationToken())
{
BulkImportResponse bulkImportResponse = null;
long totalDocumentsAdded = 0;
double totalRequestUnitsConsumed = 0;
double totalSecondsElapsed = 0;
int totalDocumentToImport = entities.Count();
while ((bulkImportResponse == null || bulkImportResponse.NumberOfDocumentsImported < totalDocumentToImport) &&
!token.IsCancellationRequested)
{
bulkImportResponse = await _bulkExecutor.BulkImportAsync(
documents : entities,
enableUpsert : true,
disableAutomaticIdGeneration : false,
maxConcurrencyPerPartitionKeyRange : null,
maxInMemorySortingBatchSize : null,
cancellationToken : token);
totalDocumentsAdded += bulkImportResponse.NumberOfDocumentsImported;
totalRequestUnitsConsumed += bulkImportResponse.TotalRequestUnitsConsumed;
totalSecondsElapsed += bulkImportResponse.TotalTimeTaken.TotalSeconds;
_logger.LogInformation(
$"added {totalDocumentsAdded}, RU consumed: {totalRequestUnitsConsumed}, elapsed: {totalSecondsElapsed} sec");
}
return((int)totalDocumentsAdded);
}
public async void IngestDocsInBulk(
SqlClientExtension client,
IChangeFeedObserverContext context,
IReadOnlyList <Document> docs,
CancellationToken cancellationToken,
Uri destinationCollectionUri)
{
DocumentCollection documentCollection = await client.GetDocumentCollectionAsync();
InitBulkExecutor(client.DocumentClient, documentCollection);
BulkImportResponse bulkImportResponse = null;
var tokenSource = new CancellationTokenSource();
var token = tokenSource.Token;
try
{
bulkImportResponse = await bulkExecutor.BulkImportAsync(
documents : docs,
enableUpsert : true,
disableAutomaticIdGeneration : true,
maxConcurrencyPerPartitionKeyRange : null,
maxInMemorySortingBatchSize : null,
cancellationToken : token);
}
catch (DocumentClientException de)
{
Trace.TraceError("Document client exception: {0}", de);
}
catch (Exception e)
{
Trace.TraceError("Exception: {0}", e);
}
}
public async Task Run(
[Queue("%QueueName%", Connection = "QueueConnectionString")] IAsyncCollector <Document> postMortemQueue,
[CosmosDBTrigger(
databaseName: "%SourceDatabase%",
collectionName: "%SourceCollection%",
ConnectionStringSetting = "SourceCosmosDB",
LeaseCollectionName = "leases",
StartFromBeginning = true,
MaxItemsPerInvocation = 10000000,
CreateLeaseCollectionIfNotExists = true)
] IReadOnlyList <Document> documents,
ILogger log)
{
if (documents != null && documents.Count > 0)
{
BulkImportResponse bulkImportResponse = null;
List <Task> tasks = new List <Task>();
tasks.Add(Task.Run(async() =>
{
try
{
bulkImportResponse = await bulkExecutor.BulkImportAsync(
documents: documents,
enableUpsert: true,
disableAutomaticIdGeneration: true,
maxInMemorySortingBatchSize: 10000000,
cancellationToken: default(CancellationToken));
}
catch (DocumentClientException e)
{
log.LogError("Document client Exception: {0}", e);
}
catch (Exception e)
{
log.LogError("Exception: {0}", e);
}
if (bulkImportResponse.BadInputDocuments != null && bulkImportResponse.BadInputDocuments.Count > 0)
{
foreach (Document doc in bulkImportResponse.BadInputDocuments)
{
await postMortemQueue.AddAsync(doc);
log.LogInformation("Document added to the post-mortem queue: {0}", doc.Id);
}
}
},
default(CancellationToken)));
await Task.WhenAll(tasks);
log.LogMetric("The Number of Documents Imported", bulkImportResponse.NumberOfDocumentsImported);
log.LogMetric("The Total Number of RU/s consumed", bulkImportResponse.TotalRequestUnitsConsumed);
log.LogMetric("RU/s per Document Write", bulkImportResponse.TotalRequestUnitsConsumed / bulkImportResponse.NumberOfDocumentsImported);
log.LogMetric("RU/s being used", bulkImportResponse.TotalRequestUnitsConsumed / bulkImportResponse.TotalTimeTaken.TotalSeconds);
log.LogMetric("Migration Time", bulkImportResponse.TotalTimeTaken.TotalMinutes);
}
}
public async Task BulkImportAsync(IEnumerable <object> documents)
{
var response = await _graphBulkExecutor.BulkImportAsync(documents, enableUpsert : true);
if (response.BadInputDocuments.Any())
{
_logger.Error("{@badInputDocuments}", response.BadInputDocuments);
throw new Exception($"GraphBulkExecutor found {response.BadInputDocuments.Count} bad graph element(s)!");
}
}
public async Task Insert <T>(string destination, IEnumerable <T> data)
{
if (_bulkExecutor == null)
{
_bulkExecutor = await _clientFactory.Create(DatabaseId, destination);
}
var typedData = data.Cast <object>();
await _bulkExecutor.BulkImportAsync(typedData, enableUpsert : true, disableAutomaticIdGeneration : true);
}
private async Task <BulkImportResponse> AddGreenTaxiTripsAsVertices(IBulkExecutor graphbulkExecutor, CancellationToken token, List <GreenTrip> greenTrips)
{
//Add Trips as Vertexes
ConvertBusinessObjects.ConvertGreenTripsToVertices(greenTrips, Vertices, Edges);
return(await graphbulkExecutor.BulkImportAsync(
Vertices,
enableUpsert : true,
disableAutomaticIdGeneration : true,
maxConcurrencyPerPartitionKeyRange : null,
maxInMemorySortingBatchSize : null,
cancellationToken : token));
}
private async Task<CosmosBulkImportSummary> BulkInsertAsync(IBulkExecutor bulkExecutor, IEnumerable<CosmosDocumentBase> documentBatch)
{
var bulkImportResponse = await bulkExecutor.BulkImportAsync(documentBatch,
disableAutomaticIdGeneration: false,
enableUpsert: true);
return new CosmosBulkImportSummary
{
NumberOfDocumentsInserted = bulkImportResponse.NumberOfDocumentsImported,
TotalRequestUnitsConsumed = bulkImportResponse.TotalRequestUnitsConsumed,
TotalTimeTaken = bulkImportResponse.TotalTimeTaken
};
}
private static async Task BulkImportToCosmosDbAsync()
{
Console.WriteLine($"Graph elements inserted until now: {_totalElements}");
var response = await _graphBulkExecutor.BulkImportAsync(
documents : _graphElementsToAdd);
if (response.BadInputDocuments.Any())
{
throw new Exception($"BulkExecutor found {response.BadInputDocuments.Count} bad input graph element(s)!");
}
_graphElementsToAdd.Clear();
}
public async Task MatchAnnouncementsToPreferencesAsync(RoomForRentAnnouncementsIntegrationEvent integrationEvent)
{
var roomForRentAnnouncementPreferences =
await _cosmosStore.Query().Where(x => x.CityId == integrationEvent.CityId && x.ServiceActive).ToListAsync();
var processedAnnouncementPreferenceIds = new HashSet <Guid>();
Parallel.ForEach(
roomForRentAnnouncementPreferences,
_parallelOptions,
roomForRentAnnouncementPreference => ProcessRoomForRentAnnouncementPreference(roomForRentAnnouncementPreference, integrationEvent, processedAnnouncementPreferenceIds));
var documents = roomForRentAnnouncementPreferences.Select(JsonConvert.SerializeObject);
await _bulkExecutor.BulkImportAsync(documents, true, true, _maxDegreeOfParallelism);
}
public async Task ProcessChangesAsync(
IChangeFeedObserverContext context,
IReadOnlyList <Document> docs,
CancellationToken cancellationToken)
{
BulkImportResponse bulkImportResponse = new BulkImportResponse();
try
{
Boolean isSyntheticKey = SourcePartitionKeys.Contains(",");
Boolean isNestedAttribute = SourcePartitionKeys.Contains("/");
List <Document> transformedDocs = new List <Document>();
Document document = new Document();
foreach (var doc in docs)
{
document = (SourcePartitionKeys != null & TargetPartitionKey != null) ? MapPartitionKey(doc, isSyntheticKey, TargetPartitionKey, isNestedAttribute, SourcePartitionKeys) : document = doc;
transformedDocs.AddRange(documentTransformer.TransformDocument(document).Result);
}
bulkImportResponse = await bulkExecutor.BulkImportAsync(
documents : transformedDocs,
enableUpsert : true,
maxConcurrencyPerPartitionKeyRange : 1,
disableAutomaticIdGeneration : true,
maxInMemorySortingBatchSize : null,
cancellationToken : new CancellationToken(),
maxMiniBatchSizeBytes : 100 * 1024);
if (bulkImportResponse.FailedImports.Count > 0 && containerClient != null)
{
WriteFailedDocsToBlob("FailedImportDocs", containerClient, bulkImportResponse);
}
if (bulkImportResponse.BadInputDocuments.Count > 0 && containerClient != null)
{
WriteFailedDocsToBlob("BadInputDocs", containerClient, bulkImportResponse);
}
LogMetrics(context, bulkImportResponse);
}
catch (Exception e)
{
Program.telemetryClient.TrackException(e);
}
Program.telemetryClient.Flush();
}
private static async Task WriteProducts(IBulkExecutor bulkExecutor, int nofProducts, int nofProperties)
{
Console.WriteLine("Writing products...");
var sw = Stopwatch.StartNew();
{
await bulkExecutor.BulkImportAsync(
ProductGenerator.GenerateProducts(nofProducts, nofProperties)
.Select(x => JObject.FromObject(x, Serializer)),
enableUpsert : true);
}
sw.Stop();
Console.WriteLine($"Written {nofProducts} products with {nofProperties} properties in: {sw.Elapsed}");
var rps = (nofProducts + nofProducts * nofProperties) / sw.Elapsed.TotalSeconds;
Console.WriteLine($"That is {rps:0.##} records/s");
}
public async Task <BulkImportResponse> BulkSendToNewCollection <T>(List <T> entityList)
{
BulkImportResponse bulkImportResponse = null;
var tokenSource = new CancellationTokenSource();
var token = tokenSource.Token;
int attempts = 0;
var objList = entityList.Cast <Object>();
do
{
bulkImportResponse = await bulkExecutor.BulkImportAsync(
documents : objList,
enableUpsert : true,
disableAutomaticIdGeneration : true,
maxConcurrencyPerPartitionKeyRange : null,
maxInMemorySortingBatchSize : null,
cancellationToken : token);
attempts++;
} while (bulkImportResponse.NumberOfDocumentsImported < entityList.Count() && attempts <= 5);
var badDocumentList = bulkImportResponse.BadInputDocuments;
#region log bulk Summary
logger.LogInfo(String.Format("\n Batch Upload completed "));
logger.LogInfo("--------------------------------------------------------------------- ");
logger.LogInfo(String.Format("Inserted {0} docs @ {1} writes/s, {2} RU/s in {3} sec",
bulkImportResponse.NumberOfDocumentsImported,
Math.Round(bulkImportResponse.NumberOfDocumentsImported / bulkImportResponse.TotalTimeTaken.TotalSeconds, 2),
Math.Round(bulkImportResponse.TotalRequestUnitsConsumed / bulkImportResponse.TotalTimeTaken.TotalSeconds, 2),
bulkImportResponse.TotalTimeTaken.TotalSeconds));
logger.LogInfo(String.Format("Average RU consumption per document: {0}",
Math.Round(bulkImportResponse.TotalRequestUnitsConsumed / bulkImportResponse.NumberOfDocumentsImported, 2)));
if (badDocumentList != null && badDocumentList.Count > 0)
{
logger.LogInfo($"bad Documents detected {badDocumentList.Count}");
}
logger.LogInfo("---------------------------------------------------------------------\n ");
#endregion
return(bulkImportResponse);
}
private async Task <BulkImportResponse> AddLocationAsVertices(IBulkExecutor graphbulkExecutor, CancellationToken token)
{
IEnumerable <Location> locations = from data in Utils.GetDataPerLines(@"C:\TaxiData\taxi+_zone_lookup.csv")
let splitChr = data.Replace("\"", "").Split(",".ToCharArray())
select new Location
{
LocationID = Convert.ToInt32(splitChr[0]),
Borough = splitChr[1],
Zone = splitChr[2],
service_zone = splitChr[3]
};
//Add Trips as Vertexes
return(await graphbulkExecutor.BulkImportAsync(
ConvertBusinessObjects.ConvertLocationsToVertices(locations),
enableUpsert : true,
disableAutomaticIdGeneration : true,
maxConcurrencyPerPartitionKeyRange : null,
maxInMemorySortingBatchSize : null,
cancellationToken : token));
}