private static async Task ImportVerticesAsync(IEnumerable <GremlinVertex> gremlinVertices)
        {
            Trace.TraceInformation(nameof(ImportVerticesAsync));
            var token = new CancellationTokenSource().Token;
            BulkImportResponse vResponse = null;

            while (Client == null || _graphBulkExecutor == null || _initializeAsyncTask?.IsCompleted != true)
            {
                await Task.Delay(100.Milliseconds(), token);
            }

            try
            {
                if (gremlinVertices != null)
                {
                    vResponse = await _graphBulkExecutor.BulkImportAsync(
                        gremlinVertices,
                        true,
                        true,
                        null,
                        null,
                        token);
                }
            }
            catch (DocumentClientException de)
            {
                Trace.TraceError("Document client exception: {0}", de);
            }
            catch (Exception e)
            {
                Trace.TraceError("Exception: {0}", e);
            }

            Trace.TraceInformation("END" + nameof(ImportVerticesAsync));
        }
예제 #2
0
        public async Task BulkInsertVertices(IEnumerable <V> vertices, string partition, CancellationToken cancel)
        {
            await InitBulkExecutor();

            var objs = vertices.Select(ToVertex).ToList();

            logger.LogInformation($"Adding vertices to partition {partition}...{objs.Count}");
            int pos           = 0;
            int totalInserted = 0;

            while (pos < objs.Count)
            {
                var batch      = objs.Skip(pos).Take(100);
                var retryCount = 0;
                var succeed    = false;
                while (!succeed && retryCount < 3)
                {
                    try
                    {
                        await bulkExecutor.BulkImportAsync(batch, true, true, null, null, cancel);

                        succeed = true;
                    }
                    catch (DocumentClientException ex) when(ex.StatusCode == (HttpStatusCode)429)
                    {
                        retryCount++;
                        Thread.Sleep(TimeSpan.FromSeconds(1));
                    }
                }

                totalInserted += batch.Count();
                logger.LogInformation($"executing batch...{totalInserted}");
                pos += batch.Count();
            }
        }
예제 #3
0
        public async Task ProcessChangesAsync(
            IChangeFeedObserverContext context,
            IReadOnlyList <Document> docs,
            CancellationToken cancellationToken)
        {
            try
            {
                List <Document> transformedDocs = new List <Document>();
                foreach (var doc in docs)
                {
                    transformedDocs.AddRange(documentTransformer.TransformDocument(doc).Result);
                }

                BulkImportResponse bulkImportResponse = await bulkExecutor.BulkImportAsync(
                    documents : transformedDocs,
                    enableUpsert : true,
                    maxConcurrencyPerPartitionKeyRange : 1,
                    disableAutomaticIdGeneration : true,
                    maxInMemorySortingBatchSize : null,
                    cancellationToken : new CancellationToken());

                LogMetrics(context, bulkImportResponse);
            }
            catch (Exception e)
            {
                Program.telemetryClient.TrackException(e);
            }

            Program.telemetryClient.Flush();
        }
예제 #4
0
        public async Task <int> BulkImport(IEnumerable <T> entities, CancellationToken token = new CancellationToken())
        {
            BulkImportResponse bulkImportResponse = null;
            long   totalDocumentsAdded            = 0;
            double totalRequestUnitsConsumed      = 0;
            double totalSecondsElapsed            = 0;
            int    totalDocumentToImport          = entities.Count();

            while ((bulkImportResponse == null || bulkImportResponse.NumberOfDocumentsImported < totalDocumentToImport) &&
                   !token.IsCancellationRequested)
            {
                bulkImportResponse = await _bulkExecutor.BulkImportAsync(
                    documents : entities,
                    enableUpsert : true,
                    disableAutomaticIdGeneration : false,
                    maxConcurrencyPerPartitionKeyRange : null,
                    maxInMemorySortingBatchSize : null,
                    cancellationToken : token);

                totalDocumentsAdded       += bulkImportResponse.NumberOfDocumentsImported;
                totalRequestUnitsConsumed += bulkImportResponse.TotalRequestUnitsConsumed;
                totalSecondsElapsed       += bulkImportResponse.TotalTimeTaken.TotalSeconds;
                _logger.LogInformation(
                    $"added {totalDocumentsAdded}, RU consumed: {totalRequestUnitsConsumed}, elapsed: {totalSecondsElapsed} sec");
            }

            return((int)totalDocumentsAdded);
        }
        public async void IngestDocsInBulk(
            SqlClientExtension client,
            IChangeFeedObserverContext context,
            IReadOnlyList <Document> docs,
            CancellationToken cancellationToken,
            Uri destinationCollectionUri)
        {
            DocumentCollection documentCollection = await client.GetDocumentCollectionAsync();

            InitBulkExecutor(client.DocumentClient, documentCollection);

            BulkImportResponse bulkImportResponse = null;
            var tokenSource = new CancellationTokenSource();
            var token       = tokenSource.Token;

            try
            {
                bulkImportResponse = await bulkExecutor.BulkImportAsync(
                    documents : docs,
                    enableUpsert : true,
                    disableAutomaticIdGeneration : true,
                    maxConcurrencyPerPartitionKeyRange : null,
                    maxInMemorySortingBatchSize : null,
                    cancellationToken : token);
            }
            catch (DocumentClientException de)
            {
                Trace.TraceError("Document client exception: {0}", de);
            }
            catch (Exception e)
            {
                Trace.TraceError("Exception: {0}", e);
            }
        }
        public async Task Run(
            [Queue("%QueueName%", Connection = "QueueConnectionString")] IAsyncCollector <Document> postMortemQueue,
            [CosmosDBTrigger(
                 databaseName: "%SourceDatabase%",
                 collectionName: "%SourceCollection%",
                 ConnectionStringSetting = "SourceCosmosDB",
                 LeaseCollectionName = "leases",
                 StartFromBeginning = true,
                 MaxItemsPerInvocation = 10000000,
                 CreateLeaseCollectionIfNotExists = true)
            ] IReadOnlyList <Document> documents,
            ILogger log)
        {
            if (documents != null && documents.Count > 0)
            {
                BulkImportResponse bulkImportResponse = null;

                List <Task> tasks = new List <Task>();

                tasks.Add(Task.Run(async() =>
                {
                    try
                    {
                        bulkImportResponse = await bulkExecutor.BulkImportAsync(
                            documents: documents,
                            enableUpsert: true,
                            disableAutomaticIdGeneration: true,
                            maxInMemorySortingBatchSize: 10000000,
                            cancellationToken: default(CancellationToken));
                    }
                    catch (DocumentClientException e)
                    {
                        log.LogError("Document client Exception: {0}", e);
                    }
                    catch (Exception e)
                    {
                        log.LogError("Exception: {0}", e);
                    }

                    if (bulkImportResponse.BadInputDocuments != null && bulkImportResponse.BadInputDocuments.Count > 0)
                    {
                        foreach (Document doc in bulkImportResponse.BadInputDocuments)
                        {
                            await postMortemQueue.AddAsync(doc);
                            log.LogInformation("Document added to the post-mortem queue: {0}", doc.Id);
                        }
                    }
                },
                                   default(CancellationToken)));

                await Task.WhenAll(tasks);

                log.LogMetric("The Number of Documents Imported", bulkImportResponse.NumberOfDocumentsImported);
                log.LogMetric("The Total Number of RU/s consumed", bulkImportResponse.TotalRequestUnitsConsumed);
                log.LogMetric("RU/s per Document Write", bulkImportResponse.TotalRequestUnitsConsumed / bulkImportResponse.NumberOfDocumentsImported);
                log.LogMetric("RU/s being used", bulkImportResponse.TotalRequestUnitsConsumed / bulkImportResponse.TotalTimeTaken.TotalSeconds);
                log.LogMetric("Migration Time", bulkImportResponse.TotalTimeTaken.TotalMinutes);
            }
        }
예제 #7
0
        public async Task BulkImportAsync(IEnumerable <object> documents)
        {
            var response = await _graphBulkExecutor.BulkImportAsync(documents, enableUpsert : true);

            if (response.BadInputDocuments.Any())
            {
                _logger.Error("{@badInputDocuments}", response.BadInputDocuments);
                throw new Exception($"GraphBulkExecutor found {response.BadInputDocuments.Count} bad graph element(s)!");
            }
        }
        public async Task Insert <T>(string destination, IEnumerable <T> data)
        {
            if (_bulkExecutor == null)
            {
                _bulkExecutor = await _clientFactory.Create(DatabaseId, destination);
            }

            var typedData = data.Cast <object>();

            await _bulkExecutor.BulkImportAsync(typedData, enableUpsert : true, disableAutomaticIdGeneration : true);
        }
예제 #9
0
 private async Task <BulkImportResponse> AddGreenTaxiTripsAsVertices(IBulkExecutor graphbulkExecutor, CancellationToken token, List <GreenTrip> greenTrips)
 {
     //Add Trips as Vertexes
     ConvertBusinessObjects.ConvertGreenTripsToVertices(greenTrips, Vertices, Edges);
     return(await graphbulkExecutor.BulkImportAsync(
                Vertices,
                enableUpsert : true,
                disableAutomaticIdGeneration : true,
                maxConcurrencyPerPartitionKeyRange : null,
                maxInMemorySortingBatchSize : null,
                cancellationToken : token));
 }
예제 #10
0
        private async Task<CosmosBulkImportSummary> BulkInsertAsync(IBulkExecutor bulkExecutor, IEnumerable<CosmosDocumentBase> documentBatch)
        {
            var bulkImportResponse = await bulkExecutor.BulkImportAsync(documentBatch,
                disableAutomaticIdGeneration: false,
                enableUpsert: true);

            return new CosmosBulkImportSummary
            {
                NumberOfDocumentsInserted = bulkImportResponse.NumberOfDocumentsImported,
                TotalRequestUnitsConsumed = bulkImportResponse.TotalRequestUnitsConsumed,
                TotalTimeTaken = bulkImportResponse.TotalTimeTaken
            };
        }
예제 #11
0
        private static async Task BulkImportToCosmosDbAsync()
        {
            Console.WriteLine($"Graph elements inserted until now: {_totalElements}");

            var response = await _graphBulkExecutor.BulkImportAsync(
                documents : _graphElementsToAdd);

            if (response.BadInputDocuments.Any())
            {
                throw new Exception($"BulkExecutor found {response.BadInputDocuments.Count} bad input graph element(s)!");
            }

            _graphElementsToAdd.Clear();
        }
예제 #12
0
        public async Task MatchAnnouncementsToPreferencesAsync(RoomForRentAnnouncementsIntegrationEvent integrationEvent)
        {
            var roomForRentAnnouncementPreferences =
                await _cosmosStore.Query().Where(x => x.CityId == integrationEvent.CityId && x.ServiceActive).ToListAsync();

            var processedAnnouncementPreferenceIds = new HashSet <Guid>();

            Parallel.ForEach(
                roomForRentAnnouncementPreferences,
                _parallelOptions,
                roomForRentAnnouncementPreference => ProcessRoomForRentAnnouncementPreference(roomForRentAnnouncementPreference, integrationEvent, processedAnnouncementPreferenceIds));

            var documents = roomForRentAnnouncementPreferences.Select(JsonConvert.SerializeObject);
            await _bulkExecutor.BulkImportAsync(documents, true, true, _maxDegreeOfParallelism);
        }
예제 #13
0
        public async Task ProcessChangesAsync(
            IChangeFeedObserverContext context,
            IReadOnlyList <Document> docs,
            CancellationToken cancellationToken)
        {
            BulkImportResponse bulkImportResponse = new BulkImportResponse();

            try
            {
                Boolean isSyntheticKey    = SourcePartitionKeys.Contains(",");
                Boolean isNestedAttribute = SourcePartitionKeys.Contains("/");

                List <Document> transformedDocs = new List <Document>();
                Document        document        = new Document();
                foreach (var doc in docs)
                {
                    document = (SourcePartitionKeys != null & TargetPartitionKey != null) ? MapPartitionKey(doc, isSyntheticKey, TargetPartitionKey, isNestedAttribute, SourcePartitionKeys) : document = doc;
                    transformedDocs.AddRange(documentTransformer.TransformDocument(document).Result);
                }

                bulkImportResponse = await bulkExecutor.BulkImportAsync(
                    documents : transformedDocs,
                    enableUpsert : true,
                    maxConcurrencyPerPartitionKeyRange : 1,
                    disableAutomaticIdGeneration : true,
                    maxInMemorySortingBatchSize : null,
                    cancellationToken : new CancellationToken(),
                    maxMiniBatchSizeBytes : 100 * 1024);

                if (bulkImportResponse.FailedImports.Count > 0 && containerClient != null)
                {
                    WriteFailedDocsToBlob("FailedImportDocs", containerClient, bulkImportResponse);
                }

                if (bulkImportResponse.BadInputDocuments.Count > 0 && containerClient != null)
                {
                    WriteFailedDocsToBlob("BadInputDocs", containerClient, bulkImportResponse);
                }

                LogMetrics(context, bulkImportResponse);
            }
            catch (Exception e)
            {
                Program.telemetryClient.TrackException(e);
            }

            Program.telemetryClient.Flush();
        }
예제 #14
0
        private static async Task WriteProducts(IBulkExecutor bulkExecutor, int nofProducts, int nofProperties)
        {
            Console.WriteLine("Writing products...");
            var sw = Stopwatch.StartNew();

            {
                await bulkExecutor.BulkImportAsync(
                    ProductGenerator.GenerateProducts(nofProducts, nofProperties)
                    .Select(x => JObject.FromObject(x, Serializer)),
                    enableUpsert : true);
            }
            sw.Stop();
            Console.WriteLine($"Written {nofProducts} products with {nofProperties} properties in: {sw.Elapsed}");
            var rps = (nofProducts + nofProducts * nofProperties) / sw.Elapsed.TotalSeconds;

            Console.WriteLine($"That is {rps:0.##} records/s");
        }
예제 #15
0
        public async Task <BulkImportResponse> BulkSendToNewCollection <T>(List <T> entityList)
        {
            BulkImportResponse bulkImportResponse = null;
            var tokenSource = new CancellationTokenSource();
            var token       = tokenSource.Token;
            int attempts    = 0;
            var objList     = entityList.Cast <Object>();

            do
            {
                bulkImportResponse = await bulkExecutor.BulkImportAsync(
                    documents : objList,
                    enableUpsert : true,
                    disableAutomaticIdGeneration : true,
                    maxConcurrencyPerPartitionKeyRange : null,
                    maxInMemorySortingBatchSize : null,
                    cancellationToken : token);

                attempts++;
            } while (bulkImportResponse.NumberOfDocumentsImported < entityList.Count() && attempts <= 5);

            var badDocumentList = bulkImportResponse.BadInputDocuments;

            #region log bulk Summary
            logger.LogInfo(String.Format("\n Batch Upload completed "));
            logger.LogInfo("--------------------------------------------------------------------- ");
            logger.LogInfo(String.Format("Inserted {0} docs @ {1} writes/s, {2} RU/s in {3} sec",
                                         bulkImportResponse.NumberOfDocumentsImported,
                                         Math.Round(bulkImportResponse.NumberOfDocumentsImported / bulkImportResponse.TotalTimeTaken.TotalSeconds, 2),
                                         Math.Round(bulkImportResponse.TotalRequestUnitsConsumed / bulkImportResponse.TotalTimeTaken.TotalSeconds, 2),
                                         bulkImportResponse.TotalTimeTaken.TotalSeconds));
            logger.LogInfo(String.Format("Average RU consumption per document: {0}",
                                         Math.Round(bulkImportResponse.TotalRequestUnitsConsumed / bulkImportResponse.NumberOfDocumentsImported, 2)));

            if (badDocumentList != null && badDocumentList.Count > 0)
            {
                logger.LogInfo($"bad Documents detected {badDocumentList.Count}");
            }

            logger.LogInfo("---------------------------------------------------------------------\n ");
            #endregion

            return(bulkImportResponse);
        }
예제 #16
0
        private async Task <BulkImportResponse> AddLocationAsVertices(IBulkExecutor graphbulkExecutor, CancellationToken token)
        {
            IEnumerable <Location> locations = from data in Utils.GetDataPerLines(@"C:\TaxiData\taxi+_zone_lookup.csv")
                                               let splitChr = data.Replace("\"", "").Split(",".ToCharArray())
                                                              select new Location
            {
                LocationID   = Convert.ToInt32(splitChr[0]),
                Borough      = splitChr[1],
                Zone         = splitChr[2],
                service_zone = splitChr[3]
            };


            //Add Trips as Vertexes
            return(await graphbulkExecutor.BulkImportAsync(
                       ConvertBusinessObjects.ConvertLocationsToVertices(locations),
                       enableUpsert : true,
                       disableAutomaticIdGeneration : true,
                       maxConcurrencyPerPartitionKeyRange : null,
                       maxInMemorySortingBatchSize : null,
                       cancellationToken : token));
        }