Beispiel #1
0
        public async Task UpdateOfferAsync_WhenUsingDbOffer_ThenDbOfferIsUpdated()
        {
            // Arrange
            var expectedThroughput = 20000;
            await _cosmonautClient.CreateDatabaseAsync(new Database { Id = _scaleableDbId },
                                                       new RequestOptions { OfferThroughput = 10000 });

            await _cosmonautClient.CreateCollectionAsync(_scaleableDbId, new DocumentCollection
            {
                Id           = _collectionName,
                PartitionKey = new PartitionKeyDefinition()
                {
                    Paths = new Collection <string>(new List <string> {
                        "/id"
                    })
                }
            });

            var offer = await _cosmonautClient.GetOfferV2ForDatabaseAsync(_scaleableDbId);

            // Act
            var newOffer     = new OfferV2(offer, expectedThroughput);
            var updatedOffer = await _cosmonautClient.UpdateOfferAsync(newOffer);

            // Assert
            updatedOffer.StatusCode.Should().Be(HttpStatusCode.OK);
            var dbOffer = await _cosmonautClient.GetOfferV2ForDatabaseAsync(_scaleableDbId);

            dbOffer.Content.OfferThroughput.Should().Be(expectedThroughput);
        }
Beispiel #2
0
        public async Task UpdateCollection(string databaseName, string collectionId, int defaultTimeToLive, int reservedRU)
        {
            try
            {
                DocumentCollection collectionInfo = await _client.ReadDocumentCollectionAsync(UriFactory.CreateDocumentCollectionUri(databaseName, collectionId));

                collectionInfo.DefaultTimeToLive = defaultTimeToLive;
                await _client.ReplaceDocumentCollectionAsync(collectionInfo);

                Offer offer = _client.CreateOfferQuery().Where(r => r.ResourceLink == collectionInfo.SelfLink).AsEnumerable().SingleOrDefault();
                offer = new OfferV2(offer, reservedRU);
                await _client.ReplaceOfferAsync(offer);
            }
            catch (DocumentClientException de)
            {
                if (de.StatusCode == HttpStatusCode.NotFound)
                {
                    throw new Exception("[DocumentDB] Database-" + databaseName + " not exist!");
                }
            }
            catch (Exception ex)
            {
                throw new Exception("[DocumentDB] Update Database-" + databaseName + ", collection-" + collectionId + " fail. Exception: " + ex.Message);
            }
        }
        private async Task <OfferV2> GetOfferV2Async(
            string targetRID,
            bool failIfNotConfigured            = true,
            CancellationToken cancellationToken = default(CancellationToken))
        {
            if (string.IsNullOrWhiteSpace(targetRID))
            {
                throw new ArgumentNullException(targetRID);
            }

            QueryDefinition queryDefinition = new QueryDefinition("select * from root r where r.offerResourceId= @targetRID");

            queryDefinition.WithParameter("@targetRID", targetRID);

            FeedIterator <OfferV2> databaseStreamIterator = this.GetOfferQueryIterator <OfferV2>(
                queryDefinition);
            OfferV2 offerV2 = await this.SingleOrDefaultAsync <OfferV2>(databaseStreamIterator);

            if (offerV2 == null &&
                failIfNotConfigured)
            {
                throw new CosmosException(HttpStatusCode.NotFound, "Throughput is not configured");
            }

            return(offerV2);
        }
Beispiel #4
0
        public static async Task <IActionResult> EvaluateRequestUnitsHandler(
            [HttpTrigger(AuthorizationLevel.Function, "get", "post", Route = null)] HttpRequest req,
            ILogger log,
            ExecutionContext context)
        {
            var config = new ConfigurationBuilder()
                         .SetBasePath(context.FunctionAppDirectory)
                         .AddJsonFile("local.settings.json", optional: true, reloadOnChange: true)
                         .AddEnvironmentVariables()
                         .Build();

            log.LogInformation("C# HTTP trigger function processed a request.");

            var alert = GetRequestUnitAlert(
                await new StreamReader(req.Body).ReadToEndAsync()
                );

            using (client = GetCosmosClient(config))
            {
                var offer = client.CreateOfferQuery().Where(r => r.ResourceLink == collection.SelfLink).Single();

                var offer = new OfferV2(offer, newthroughput);
                client.ReplaceOfferAsync(offer);
            }

            return(name != null
                ? (ActionResult) new OkObjectResult($"Parsed")
                : new BadRequestObjectResult("Bad request."));
        }
Beispiel #5
0
        private async Task <OfferV2> GetOfferV2Async(
            string targetRID,
            bool failIfNotConfigured,
            CancellationToken cancellationToken)
        {
            if (string.IsNullOrWhiteSpace(targetRID))
            {
                throw new ArgumentNullException(targetRID);
            }

            QueryDefinition queryDefinition = new QueryDefinition("select * from root r where r.offerResourceId= @targetRID");

            queryDefinition.WithParameter("@targetRID", targetRID);

            FeedIterator <OfferV2> databaseStreamIterator = this.GetOfferQueryIterator <OfferV2>(
                queryDefinition: queryDefinition,
                continuationToken: null,
                requestOptions: null,
                cancellationToken: cancellationToken);
            OfferV2 offerV2 = await this.SingleOrDefaultAsync <OfferV2>(databaseStreamIterator);

            if (offerV2 == null &&
                failIfNotConfigured)
            {
                throw (CosmosException)CosmosExceptionFactory.CreateNotFoundException(
                          $"Throughput is not configured for {targetRID}");
            }

            return(offerV2);
        }
        private async Task CheckScaling(bool force)
        {
            if (Options.ScaleOfferThroughput || force)
            {
                var client = CreateClient();

                var col = client.CreateDocumentCollectionQuery(UriFactory.CreateDatabaseUri(Options.DatabaseId))
                          .Where(x => x.Id == Options.CollectionId)
                          .AsEnumerable()
                          .SingleOrDefault()
                ;

                var offer =
                    client.CreateOfferQuery()
                    .Where(r => r.ResourceLink == col.SelfLink)
                    .AsEnumerable()
                    .SingleOrDefault();

                _logger.LogInformation("Setting OfferThroughput to {0}", Options.OfferThroughput);
                // Set the throughput to the new value, for example 12,000 request units per second
                offer = new OfferV2(offer, Options.OfferThroughput);

                // Now persist these changes to the collection by replacing the original offer resource
                await client.ReplaceOfferAsync(offer);
            }
        }
        private static async Task GetAndChangeCollectionPerformance(DocumentCollection simpleCollection)
        {
            //*********************************************************************************************
            // Get configured performance (reserved throughput) of a DocumentCollection
            //
            //    DocumentCollections each have a corresponding Offer resource that represents the reserved throughput of the collection.
            //    Offers are "linked" to DocumentCollection through the collection's SelfLink (Offer.ResourceLink == Collection.SelfLink)
            //
            //**********************************************************************************************
            Offer offer = client.CreateOfferQuery().Where(o => o.ResourceLink == simpleCollection.SelfLink).AsEnumerable().Single();

            Console.WriteLine("\n2. Found Offer \n{0}\nusing collection's SelfLink \n{1}", offer, simpleCollection.SelfLink);

            //******************************************************************************************************************
            // Change performance (reserved throughput) of DocumentCollection
            //    Let's change the performance of the collection to 500 RU/s
            //******************************************************************************************************************

            Offer replaced = await client.ReplaceOfferAsync(new OfferV2(offer, 500));

            Console.WriteLine("\n3. Replaced Offer. Offer is now {0}.\n", replaced);

            // Get the offer again after replace
            offer = client.CreateOfferQuery().Where(o => o.ResourceLink == simpleCollection.SelfLink).AsEnumerable().Single();
            OfferV2 offerV2 = (OfferV2)offer;

            Console.WriteLine(offerV2.Content.OfferThroughput);

            Console.WriteLine("3. Found Offer \n{0}\n using collection's ResourceId {1}.\n", offer, simpleCollection.ResourceId);
        }
Beispiel #8
0
        public async override Task <ThroughputResponse> ReplaceThroughputAsync(
            int throughput,
            RequestOptions requestOptions       = null,
            CancellationToken cancellationToken = default(CancellationToken))
        {
            string rid = await this.GetRIDAsync(cancellationToken);

            OfferV2 offerV2 = await this.ClientContext.Client.Offers.GetOfferV2Async(rid, cancellationToken);

            if (offerV2 == null)
            {
                return(new ThroughputResponse(httpStatusCode: HttpStatusCode.NotFound, headers: null, throughputProperties: null));
            }

            OfferV2 newOffer = new OfferV2(offerV2, throughput);

            Task <ResponseMessage> response = this.ProcessResourceOperationStreamAsync(
                streamPayload: this.ClientContext.PropertiesSerializer.ToStream(newOffer),
                operationType: OperationType.Replace,
                linkUri: new Uri(offerV2.SelfLink, UriKind.Relative),
                resourceType: ResourceType.Offer,
                requestOptions: requestOptions,
                cancellationToken: cancellationToken);

            return(await this.ClientContext.ResponseFactory.CreateThroughputResponseAsync(response));
        }
Beispiel #9
0
        /// <summary>
        /// Updates the throughput for the specified collection.
        /// </summary>
        /// <param name="offerThroughput">The provisioned throughtput for this offer.</param>
        /// <returns>An instance of the <see cref="Task" /> class that represents the asynchronous operation.</returns>
        private async Task UpdateOfferAsync(int offerThroughput)
        {
            DocumentCollection collection;
            Offer offer;

            try
            {
                collection = await Client.ReadDocumentCollectionAsync(
                    UriFactory.CreateDocumentCollectionUri(databaseId, collectionId)).ConfigureAwait(false);

                offer = Client.CreateOfferQuery()
                        .Where(o => o.ResourceLink == collection.SelfLink)
                        .AsEnumerable()
                        .SingleOrDefault();

                offer = new OfferV2(offer, offerThroughput);

                await Client.ReplaceOfferAsync(offer).ConfigureAwait(false);
            }
            finally
            {
                collection = null;
                offer      = null;
            }
        }
        private async Task <int> SetCollectionRU(int targetRU)
        {
            var collectionUri = UriFactory.CreateDocumentCollectionUri(database, collection).ToString();

            // to query for a collection's offer, we first need the collection object
            // from the database, which will contain the (needed) self-link.
            DocumentCollection coll = await client.ReadDocumentCollectionAsync(collectionUri);

            // Note: The "Offer" class does not expose throughput as a property (it's a legacy
            // class). The new "OfferV2" class *does* expose it, and we can cast the returned Offer
            // class to OfferV2. This allows us to capture the collection's current throughput,
            // before scaling to a different number.
            var offer = (OfferV2)client.CreateOfferQuery()
                        .Where(r => r.ResourceLink == coll.SelfLink)
                        .AsEnumerable()
                        .SingleOrDefault();

            var currentRU = offer.Content.OfferThroughput;

            Console.WriteLine("Changing RU from {0} to {1}", currentRU, targetRU);

            // we cannot just adjust the current offer's throughput, as the property is
            // read-only. We have to create a new instance, for the purpose of scaling.
            offer = new OfferV2(offer, targetRU);
            await client.ReplaceOfferAsync(offer);

            return(currentRU);
        }
        public async Task <Offer> UpdateOfferForCollectionAsync(string collectionSelfLink, int newOfferThroughput)
        {
            // Create an asynchronous query to retrieve the current offer for the document collection
            // Notice that the current version of the API only allows to use the SelfLink for the collection
            // to retrieve its associated offer
            Offer existingOffer = null;
            var   offerQuery    = DocumentDbClientInstance.Client.CreateOfferQuery()
                                  .Where(o => o.ResourceLink == collectionSelfLink)
                                  .AsDocumentQuery();

            while (offerQuery.HasMoreResults)
            {
                foreach (var offer in await offerQuery.ExecuteNextAsync <Offer>())
                {
                    existingOffer = offer;
                }
            }
            if (existingOffer == null)
            {
                throw new Exception("I couldn't retrieve the offer for the collection.");
            }
            // Set the desired throughput to newOfferThroughput RU/s for the new offer built based on the current offer
            var newOffer             = new OfferV2(existingOffer, newOfferThroughput);
            var replaceOfferResponse = await DocumentDbClientInstance.Client.ReplaceOfferAsync(newOffer);

            return(replaceOfferResponse.Resource);
        }
Beispiel #12
0
        public static void IncreaseRUs(DocumentClient client, string databaseId, string collectionId,
                                       bool sharedOfferThroughput = false, int rUsOfferThroughputMax = 1000, int rUsScaleStepUp = 200, TimeSpan?retryAfter = null)
        {
            try
            {
                Uri    dataBaseOrCollectionLink;
                string selfLink;

                if (sharedOfferThroughput)
                {
                    dataBaseOrCollectionLink = UriFactory.CreateDatabaseUri(databaseId);
                    var resourceResponse       = client.ReadDatabaseAsync(dataBaseOrCollectionLink).Result;
                    var dbOrCollectionResource = resourceResponse.Resource;
                    selfLink = dbOrCollectionResource.SelfLink;
                }
                else
                {
                    dataBaseOrCollectionLink = UriFactory.CreateDocumentCollectionUri(databaseId, collectionId);
                    var resourceResponse       = client.ReadDocumentCollectionAsync(dataBaseOrCollectionLink).Result;
                    var dbOrCollectionResource = resourceResponse.Resource;
                    selfLink = dbOrCollectionResource.SelfLink;
                }

                var offer = client.CreateOfferQuery()
                            .Where(o => o.ResourceLink == selfLink)
                            .AsDocumentQuery()
                            .ExecuteNextAsync <OfferV2>().Result.FirstOrDefault();

                if (offer != null)
                {
                    var currentOfferThroughput = offer.Content.OfferThroughput;

                    Trace.WriteLine($"currentOfferThroughput: {currentOfferThroughput} ");

                    if (currentOfferThroughput < rUsOfferThroughputMax)
                    {
                        var start = DateTime.Now;
                        var newOfferThroughput = currentOfferThroughput + rUsScaleStepUp;
                        var updatedOffer       = new OfferV2(offer, offerThroughput: newOfferThroughput);
                        var offerAfterUpdate   = (OfferV2)client.ReplaceOfferAsync(updatedOffer).Result.Resource;
                        var end  = DateTime.Now;
                        var span = end - start;

                        Trace.WriteLine($"OldOfferThroughput: {currentOfferThroughput}   NewOfferThroughput: {offerAfterUpdate.Content.OfferThroughput} take {span} ms");
                    }
                    else
                    {
                        if (retryAfter != null)
                        {
                            Trace.WriteLine("RetryAfter: " + retryAfter);
                            Task.Delay((TimeSpan)retryAfter);
                        }
                    }
                }
            }
            catch (Exception)
            {
            }
        }
Beispiel #13
0
        private static async void ScaleCosmosDB(string databaseId, string collectionId, string databaseUri, string databaseKey, ILogger log, int newThroughput)
        {
            try
            {
                //1) initialize the document client
                using (DocumentClient client = new DocumentClient(new Uri(databaseUri), databaseKey))
                {
                    //2) get the database self link
                    string selfLink = client.CreateDocumentCollectionQuery(
                        UriFactory.CreateDatabaseUri(databaseId))
                                      .Where(c => c.Id == collectionId)
                                      .AsEnumerable()
                                      .FirstOrDefault()
                                      .SelfLink;

                    //3) get the current offer for the collection
                    Offer offer = client.CreateOfferQuery()
                                  .Where(r => r.ResourceLink == selfLink)
                                  .AsEnumerable()
                                  .SingleOrDefault();

                    //4) get the current throughput from the offer
                    int throughputCurrent = (int)offer.GetPropertyValue <JObject>("content").GetValue("offerThroughput");
                    log.LogInformation($"Current provisioned throughput of Database:{databaseId}, Collection: {collectionId} is: {throughputCurrent.ToString()} RU.");

                    //5) get the RU increment from AppSettings and parse to an int
                    if (int.TryParse(ConfigurationManager.AppSettings["cosmosDB_RUIncrement"], out int RUIncrement))
                    {
                        //5.a) create the new offer with the throughput increment added to the current throughput
                        if (newThroughput == 0)
                        {
                            newThroughput = throughputCurrent + RUIncrement;
                        }
                        else if (newThroughput < 0)
                        {
                            newThroughput = throughputCurrent - RUIncrement;
                        }

                        offer = new OfferV2(offer, newThroughput);

                        //5.b) persist the changes
                        await client.ReplaceOfferAsync(offer);

                        log.LogInformation($"New provisioned throughput of Database:{databaseId}, Collection: {collectionId} is: {newThroughput} RU.");
                    }
                    else
                    {
                        //5.c) if the throughputIncrement cannot be parsed return throughput not changed
                        throw new Exception("Throughput not changed!");
                    }
                }
            }
            catch (Exception e)
            {
                log.LogError(e.Message);
                throw;
            }
        }
        private static async Task VerifyCollectionThroughput(this DocumentClient client, DocumentCollection dataCollection, int collectionThroughput)
        {
            OfferV2 offer = (OfferV2)client.CreateOfferQuery().Where(o => o.ResourceLink == dataCollection.SelfLink).AsEnumerable().FirstOrDefault();

            if (collectionThroughput != offer.Content.OfferThroughput)
            {
                await client.ReplaceOfferAsync(new OfferV2(offer, collectionThroughput));
            }
        }
Beispiel #15
0
        public async Task <OfferV2> ReplaceOfferAsync(OfferV2 offer, int requestUnits)
        {
            var newOffer = new OfferV2(
                offer,
                requestUnits);
            var response = await _client.ReplaceOfferAsync(newOffer);

            return(response.Resource as OfferV2);
        }
Beispiel #16
0
        private async void ChangeCosmosRu(string databaseAccount, string primaryKey, string databasename, Collection coll, int pkr, double averageRu)
        {
            try
            {
                var endPointUrl    = $"https://{databaseAccount}.documents.azure.cn:443";
                var documentClient = new DocumentClient(new Uri(endPointUrl), primaryKey);
                Microsoft.Azure.Documents.Database database = await GetOrCreateDatabaseAsync(documentClient,
                                                                                             databasename);

                DocumentCollection collection = await GetOrCreateCollectionAsync(documentClient, database.SelfLink,
                                                                                 coll.Name);

                Offer offer = documentClient.CreateOfferQuery()
                              .Where(r => r.ResourceLink == collection.SelfLink)
                              .AsEnumerable()
                              .SingleOrDefault();
                int offerThroughput   = JsonConvert.DeserializeObject <dynamic>(offer.ToString()).content.offerThroughput; //当前设置的RU
                var currentThroughput = pkr * averageRu;                                                                   //当前实时RU
                var ruRate            = Math.Round(currentThroughput / offerThroughput, 2);                                //计算实时RU和设置的RU占
                if ((ruRate - coll.IncreaseRate >= coll.ThresholdRate || ruRate + coll.IncreaseRate < coll.ThresholdRate) && currentThroughput > 1000)
                {
                    var newOfferThroughput = Convert.ToInt32(currentThroughput / coll.ThresholdRate); //保证RU为阈值左右
                    var increaseValue      = Math.Abs(newOfferThroughput - currentThroughput);        //计算调整RU的差值
                    if (increaseValue > 100)
                    {
                        newOfferThroughput = newOfferThroughput - newOfferThroughput % 100 + 100;
                        if (newOfferThroughput < coll.DefaultValue)
                        {
                            newOfferThroughput = coll.DefaultValue;
                        }
                        offer = new OfferV2(offer, newOfferThroughput);
                        Console.WriteLine(databasename + "-" + coll.Name + ",当前设置" + offerThroughput + "RU" + ",当前消耗" +
                                          currentThroughput + "RU" + ",调整设置为" + newOfferThroughput + "RU");
                        await documentClient.ReplaceOfferAsync(offer);
                    }
                    else
                    {
                        Console.WriteLine(databasename + "-" + coll.Name + ",当前设置" + offerThroughput + "RU" + ",当前消耗" +
                                          currentThroughput + "RU" + ",无需调整设置,调整幅度最小为100RU");
                    }
                }
                else
                {
                    Console.WriteLine(databasename + "-" + coll.Name + ",当前设置" + offerThroughput + "RU" + ",当前消耗" +
                                      currentThroughput + "RU" + ",无需调整设置");
                }
            }
            catch (Exception ex)
            {
                string sErrorMsg = ex.Message;
                if (ex.InnerException != null)
                {
                    sErrorMsg += ex.InnerException.Message;
                }
                Console.WriteLine("ChangeCosmosRU-" + sErrorMsg);
            }
        }
Beispiel #17
0
        private async Task RunAsync()
        {
            // Blob storage client
            CloudStorageAccount storageAccount = CloudStorageAccount.Parse(ConfigurationManager.AppSettings["StorageConnectionString"]);
            CloudBlobClient     blobClient     = storageAccount.CreateCloudBlobClient();
            CloudBlobContainer  container      = blobClient.GetContainerReference(containerSourceName);
            BlobRequestOptions  requestOptions = new BlobRequestOptions()
            {
                RetryPolicy = new ExponentialRetry(TimeSpan.FromSeconds(2), 3)
            };

            // CosmosDB client
            DocumentCollection dataCollection = GetCollectionIfExists(DatabaseName, DataCollectionName);
            OfferV2            offer          = (OfferV2)docClient.CreateOfferQuery().Where(o => o.ResourceLink == dataCollection.SelfLink).AsEnumerable().FirstOrDefault();

            currentCollectionThroughput = offer.Content.OfferThroughput;

            int degreeOfParallelism = int.Parse(ConfigurationManager.AppSettings["DegreeOfParallelism"]);
            int taskCount;

            if (degreeOfParallelism == -1)
            {
                // set TaskCount = 10 for each 10k RUs, minimum 1, maximum 250
                taskCount = Math.Max(currentCollectionThroughput / 1000, 1);
                taskCount = Math.Min(taskCount, 250);
            }
            else
            {
                taskCount = degreeOfParallelism;
            }


            // List all blobs in a day
            BlobContinuationToken continuationToken = null;
            bool useFlatBlobListing = true;
            BlobListingDetails blobListingDetails = BlobListingDetails.None;
            // blob per request determined by parallelism
            int maxBlobsPerRequest        = 73000 / taskCount;
            List <IListBlobItem> listBlob = new List <IListBlobItem>();

            var dir = container.GetDirectoryReference(startDate.ToString("yyyyMMdd"));

            var tasks = new List <Task>();

            do
            {
                var listResult = await dir.ListBlobsSegmentedAsync(useFlatBlobListing, blobListingDetails, maxBlobsPerRequest, continuationToken, null, null);

                continuationToken = listResult.ContinuationToken;
                listBlob.AddRange(listResult.Results);

                tasks.Add(this.InsertDocument(blobClient, docClient, dataCollection, listResult.Results));
            }while (continuationToken != null);

            await Task.WhenAll(tasks);
        }
Beispiel #18
0
        public static async Task <IActionResult> Run(
            [HttpTrigger(AuthorizationLevel.Function, "get", "post", Route = null)] HttpRequest req,
            ILogger log,
            ExecutionContext context)
        {
            try
            {
                var config = new ConfigurationBuilder().SetBasePath(context.FunctionAppDirectory)
                             .AddJsonFile("local.settings.json", optional: true, reloadOnChange: true)
                             .AddEnvironmentVariables()
                             .Build();

                //1) initialize the cosmosdb client
                using (DocumentClient client = new DocumentClient(new Uri(config["CosmosDB_Uri"]), config["CosmosDB_appKey"]))
                {
                    //2) Get the database self link
                    string selfLink = client.CreateDocumentCollectionQuery(
                        UriFactory.CreateDatabaseUri(config["CosmosDB_DatabaseId"]))
                                      .Where(c => c.Id == config["CosmosDB_ContainerId"])
                                      .AsEnumerable()
                                      .FirstOrDefault()
                                      .SelfLink;

                    //3) Get the current offer for the collection
                    Offer offer = client.CreateOfferQuery().Where(r => r.ResourceLink == selfLink).AsEnumerable().SingleOrDefault();

                    //4) Get the current throughput from the offer
                    int throughputCurrent = (int)offer.GetPropertyValue <JObject>("content").GetValue("offerThroughput");
                    log.LogInformation(string.Format("Current provisioned throughput is: {0} RU", throughputCurrent.ToString()));

                    //5) Get the RU increment from AppSettings and parse to an int
                    if (int.TryParse(config["CosmosDB_RU"], out int RUIncrement))
                    {
                        //5.a) create the new offer with the throughput increment added to the current throughput
                        int newThroughput = throughputCurrent + RUIncrement;
                        offer = new OfferV2(offer, newThroughput);

                        //5.b) persist the changes
                        await client.ReplaceOfferAsync(offer);

                        log.LogInformation(string.Format("New provisioned througput: {0} RU", newThroughput.ToString()));
                        return(new OkObjectResult("The collection's throughput was changed..."));
                    }
                    else
                    {
                        //5.c) if the throughputIncrement cannot be parsed return throughput not changed
                        return(new BadRequestObjectResult("PARSE ERROR: The collection's throughput was not changed..."));
                    }
                }
            }
            catch (Exception e)
            {
                log.LogInformation(e.Message);
                return(new BadRequestObjectResult("ERROR: The collection's throughput was not changed..."));
            }
        }
            private static async Task <Offer> ReplaceOffer(DocumentClient client, Offer currentOffer, int newThroughputOffer)
            {
                //Potentially the new throughput may have been calculated using a factional multiplier.
                //CosmosDB throughputs must be specified in hundred units.
                newThroughputOffer = RoundSignaficantFigures(newThroughputOffer, 2);

                OfferV2 replacementOffer = new OfferV2(currentOffer, newThroughputOffer);

                return(await client.ReplaceOfferAsync(replacementOffer));
            }
Beispiel #20
0
        /// <summary>
        /// Run the main body of the program: create the database and collection, if they do not exist,
        /// then run the specified experiment,
        /// cleaning up afterwards if configured.
        /// </summary>
        /// <returns>The task</returns>
        private async Task RunAsync()
        {
            Database database = GetDatabaseIfExists(Options.Database);

            if (bool.Parse(ConfigurationManager.AppSettings["ShouldCleanupOnStart"]) && database != null)
            {
                Console.WriteLine("Deleting database {0}", Options.Database);
                await client.DeleteDatabaseAsync(database.SelfLink);
            }

            if (bool.Parse(ConfigurationManager.AppSettings["ShouldCleanupOnStart"]) || database == null)
            {
                Console.WriteLine("Creating database {0}", Options.Database);
                database = await client.CreateDatabaseAsync(new Database { Id = Options.Database });
            }

            DocumentCollection dataCollection = GetCollectionIfExists(Options.Database, Options.Collection);

            if (dataCollection == null)
            {
                Console.WriteLine("Creating collection {0} with {1} RU/s", Options.Collection, Options.Throughput);
                dataCollection = await this.CreatePartitionedCollectionAsync(Options.Database, Options.Collection, Options.Throughput, Options.PartitionKey);
            }

            long currentCollectionThroughput = 0;

            currentCollectionThroughput = Options.Throughput;

            OfferV2 offer = (OfferV2)client.CreateOfferQuery().Where(o => o.ResourceLink == dataCollection.SelfLink).AsEnumerable().FirstOrDefault();

            currentCollectionThroughput = offer.Content.OfferThroughput;

            Uri collectionUri = UriFactory.CreateDocumentCollectionUri(Options.Database, Options.Collection);

            if (Options.Verbose)
            {
                Console.WriteLine("Summary:");
                Console.WriteLine("--------------------------------------------------------------------- ");
                Console.WriteLine("Endpoint: {0}", client.ServiceEndpoint);
                Console.WriteLine("Collection : {0}.{1} at {2} RU/s with partition key {3}", Options.Database, Options.Collection, currentCollectionThroughput, Options.PartitionKey);
                Console.WriteLine("Operations : {0} {1}", Options.NumberOfOperations, Options.Operation);
                Console.WriteLine("Degree of parallelism*: {0}", Options.Parallelism);
                Console.WriteLine("--------------------------------------------------------------------- ");
                Console.WriteLine();
            }

            var experiment = new Experiment(client, dataCollection, collectionUri, Options);
            await experiment.RunAsync();

            if (bool.Parse(ConfigurationManager.AppSettings["ShouldCleanupOnFinish"]))
            {
                Console.WriteLine("Deleting Database {0}", Options.Database);
                await client.DeleteDatabaseAsync(UriFactory.CreateDatabaseUri(Options.Database));
            }
        }
        private async Task SetMaxThroughputAsync(DocumentCollection collection)
        {
            FeedResponse <PartitionKeyRange> pkRanges = await this.Client.ReadPartitionKeyRangeFeedAsync(collection.SelfLink);

            int maxThroughput = pkRanges.Count * 10000;

            Offer offer = this.Client.CreateOfferQuery().Where(o => o.ResourceLink == collection.SelfLink).AsEnumerable().FirstOrDefault();

            OfferV2 newOffer = new OfferV2(offer, maxThroughput);

            await this.Client.ReplaceOfferAsync(newOffer);
        }
Beispiel #22
0
        public static async Task <ScaleOperation> ScaleDownCollectionAsync(DocumentClient client, IMetaDataOperator metaDataOperator, string databaseName, string collectionName, int minRu)
        {
            try
            {
                Database database = client.CreateDatabaseQuery($"SELECT * FROM d WHERE d.id = \"{databaseName}\"").AsEnumerable().First();

                List <DocumentCollection> collections = client.CreateDocumentCollectionQuery((String)database.SelfLink).ToList();

                foreach (var collection in collections)
                {
                    if (collection.Id == collectionName)
                    {
                        var currentRu = GetCurrentRU(client, collection, out OfferV2 offer);

                        if (currentRu <= minRu)
                        {
                            return(new ScaleOperation()
                            {
                                ScaledSuccess = false,
                                ScaleFailReason = "RU already at minimum."
                            });
                        }

                        offer = new OfferV2(offer, minRu);

                        await client.ReplaceOfferAsync(offer);

                        Trace.WriteLine($"Scaled {databaseName}|{collectionName} to {(int)minRu}RU. ({DateTimeOffset.Now})");

                        return(new ScaleOperation()
                        {
                            ScaledTo = minRu,
                            ScaledSuccess = true,
                            OperationTime = DateTimeOffset.Now
                        });
                    }
                }

                return(new ScaleOperation()
                {
                    ScaledSuccess = false,
                    ScaleFailReason = "Collection not found."
                });
            } catch (Exception e)
            {
                return(new ScaleOperation()
                {
                    ScaledSuccess = false,
                    ScaleFailReason = e.Message
                });
            }
        }
        public static async Task <HttpResponseMessage> Run([HttpTrigger(AuthorizationLevel.Function, "get", "post", Route = null)] HttpRequestMessage req, TraceWriter log)
        {
            try
            {
                //1) initialize the document client
                using (DocumentClient client = new DocumentClient(new Uri("https://<ACCOUNTNAME>.documents.azure.com:443/"), "<PRIMARYKEY>"))
                {
                    Console.Write(client.ServiceEndpoint);
                    //2) get the database self link
                    string selfLink = client.CreateDocumentCollectionQuery(
                        UriFactory.CreateDatabaseUri(GetEnvironmentVariable("CosmosDB_DatabaseId")))
                                      .Where(c => c.Id == GetEnvironmentVariable("CosmosDB_CollectionId"))
                                      .AsEnumerable()
                                      .FirstOrDefault()
                                      .SelfLink;

                    //3) get the current offer for the collection
                    Offer offer = client.CreateOfferQuery()
                                  .Where(r => r.ResourceLink == selfLink)
                                  .AsEnumerable()
                                  .SingleOrDefault();

                    //4) get the current throughput from the offer
                    int throughputCurrent = (int)offer.GetPropertyValue <JObject>("content").GetValue("offerThroughput");
                    log.Info(string.Format("Current provisioned throughput: {0} RU", throughputCurrent.ToString()));

                    //5) get the RU increment from AppSettings and parse to an int
                    if (int.TryParse(GetEnvironmentVariable("CosmosDB_RUIncrement"), out int RUIncrement))
                    {
                        //5.a) create the new offer with the throughput increment added to the current throughput
                        int newThroughput = throughputCurrent + RUIncrement;
                        offer = new OfferV2(offer, newThroughput);

                        //5.b) persist the changes
                        await client.ReplaceOfferAsync(offer);

                        log.Info(string.Format("New provisioned througput: {0} RU", newThroughput.ToString()));
                        return(req.CreateResponse(HttpStatusCode.OK, "The collection's throughput was changed..."));
                    }
                    else
                    {
                        //5.c) if the throughputIncrement cannot be parsed return throughput not changed
                        return(req.CreateResponse(HttpStatusCode.OK, "PARSE ERROR: The collection's throughput was not changed..."));
                    }
                }
            }
            catch (Exception e)
            {
                log.Info(e.Message + e.InnerException + e.StackTrace);
                return(req.CreateResponse(HttpStatusCode.OK, e.Message + e.InnerException + e.StackTrace));
            }
        }
        private async Task <int> SetupDb()
        {
            // Determine if DB/Collection Exists
            _dataCollection = GetCollectionIfExists(_settings.DatabaseName, _settings.CollectionName);
            int currentCollectionThroughput = 0;

            // Remove DB if cleanup flag is set
            if (_settings.ShouldCleanupOnStart || _dataCollection == null)
            {
                // Assert DB does not exist
                _database = GetDatabaseIfExists(_settings.DatabaseName);
                if (_database != null)
                {
                    await DropDatabase();
                }

                // Create Database
                _logger.LogWarning("Creating database   ==> {0}", _settings.DatabaseName);
                _database = await _client.CreateDatabaseAsync(new Database { Id = _settings.DatabaseName });

                // Create Collection
                _logger.LogWarning("Creating collection ==> {0} with {1} RU/s", _settings.CollectionName, _settings.CollectionThroughput);
                _dataCollection = await this.CreatePartitionedCollectionAsync(_settings.DatabaseName, _settings.CollectionName);

                // Set throughput based upon setting
                currentCollectionThroughput = _settings.CollectionThroughput;
            }
            else
            {
                // Get Existing DB & Collection
                OfferV2 offer = (OfferV2)_client.CreateOfferQuery().Where(o => o.ResourceLink == _dataCollection.SelfLink).AsEnumerable().FirstOrDefault();
                currentCollectionThroughput = offer.Content.OfferThroughput;

                if (_settings.CollectionThroughput != currentCollectionThroughput)
                {
                    _logger.LogWarning($"Configured Throughput in Settings [{_settings.CollectionThroughput}] does not match CosmosDb setting of [{currentCollectionThroughput}]");
                    Console.WriteLine("Press any key to modify the settings...");
                    Console.ReadLine();

                    // Update the Offer to match the Throughput Number
                    offer = new OfferV2(offer, _settings.CollectionThroughput);
                    await _client.ReplaceOfferAsync(offer);

                    var updatedOffer = (OfferV2)_client.CreateOfferQuery().Where(o => o.ResourceLink == _dataCollection.SelfLink).AsEnumerable().FirstOrDefault();
                    currentCollectionThroughput = updatedOffer.Content.OfferThroughput;
                }

                currentCollectionThroughput = offer.Content.OfferThroughput;
                Console.WriteLine("Found collection ==> {0} with {1} RU/s", _settings.CollectionName, currentCollectionThroughput);
            }
            return(currentCollectionThroughput);
        }
Beispiel #25
0
        internal async Task <CosmosOfferResult> ReplaceThroughputIfExistsAsync(
            string targetRID,
            int targetThroughput,
            CancellationToken cancellationToken = default(CancellationToken))
        {
            try
            {
                Offer offer = await this.ReadOfferAsync(targetRID, cancellationToken);

                if (offer == null)
                {
                    throw new ArgumentOutOfRangeException("Throughput is not configured");
                }

                OfferV2 offerV2 = offer as OfferV2;
                if (offerV2 == null)
                {
                    throw new NotImplementedException();
                }

                OfferV2 newOffer      = new OfferV2(offerV2, targetThroughput);
                Offer   replacedOffer = await this.ReplaceOfferAsync(targetRID, newOffer, cancellationToken);

                offerV2 = replacedOffer as OfferV2;
                Debug.Assert(offerV2 != null);

                return(new CosmosOfferResult(offerV2.Content.OfferThroughput));
            }
            catch (DocumentClientException dce)
            {
                return(new CosmosOfferResult(
                           dce.StatusCode ?? HttpStatusCode.InternalServerError,
                           new CosmosException(
                               dce.Message?.Replace(Environment.NewLine, string.Empty),
                               dce.StatusCode ?? HttpStatusCode.InternalServerError,
                               (int)dce.GetSubStatus(),
                               dce.ActivityId,
                               dce.RequestCharge)));
            }
            catch (AggregateException ex)
            {
                CosmosOfferResult offerResult = CosmosOffers.TryToOfferResult(ex);
                if (offerResult != null)
                {
                    return(offerResult);
                }

                throw;
            }
        }
Beispiel #26
0
        public static void Run([TimerTrigger("0 */5 * * * *")] TimerInfo myTimer, TraceWriter log)
        {
            log.Info("Let's get this CosmosDB Collection rescaled! " + DateTime.Now.ToString());

            Decimal minRequestUnits = 400;
            Decimal requestunits    = minRequestUnits;

            // CosmosDB Parameters, retrieved via environment variables
            string host           = Environment.GetEnvironmentVariable("cosmosdbHostName");
            string password       = Environment.GetEnvironmentVariable("cosmosdbPassword");
            string databaseName   = Environment.GetEnvironmentVariable("cosmosdbDatabaseName");
            string collectionName = Environment.GetEnvironmentVariable("cosmosdbCollectionName");

            // This endpoint is valid for all APIs (tested on DocDB/SQL/MongoDB/...)
            string         endpoint    = string.Format("https://{0}:443/", host);
            Uri            endpointUri = new Uri(endpoint);
            DocumentClient client      = new DocumentClient(endpointUri, password);

            // Find links to DB Account & Collection in order to match the Offer
            Database           database       = client.CreateDatabaseQuery().Where(d => d.Id == databaseName).AsEnumerable().FirstOrDefault();
            string             databaseLink   = database.SelfLink;
            DocumentCollection collection     = client.CreateDocumentCollectionQuery(databaseLink).Where(c => c.Id == collectionName).AsEnumerable().FirstOrDefault();
            string             collectionLink = collection.SelfLink;
            string             collectionRid  = collection.GetPropertyValue <string>("_rid");

            // Loop through offers
            var offersFeed = client.CreateOfferQuery().AsEnumerable().ToArray();

            if (offersFeed != null)
            {
                foreach (var offer in offersFeed)
                {
                    var    offerColl          = client.ReadDocumentCollectionAsync(offer.ResourceLink);
                    string offerCollectionRid = offerColl.Result.Resource.GetPropertyValue <string>("_rid");

                    // Change matching offer to newly requested Request Units
                    if (offerCollectionRid == collectionRid)
                    {
                        Offer newOffer = client.CreateOfferQuery()
                                         .Where(r => r.ResourceLink == collection.SelfLink)
                                         .AsEnumerable()
                                         .SingleOrDefault();
                        newOffer = new OfferV2(newOffer, Convert.ToInt16(requestunits));
                        client.ReplaceOfferAsync(newOffer);
                        log.Info("Reset request units to " + requestunits.ToString() + "RU");
                    }
                }
            }
        }
Beispiel #27
0
        internal async Task <ThroughputResponse> ReadThroughputAsync(
            string targetRID,
            RequestOptions requestOptions,
            CancellationToken cancellationToken = default(CancellationToken))
        {
            OfferV2 offerV2 = await this.GetOfferV2Async <OfferV2>(targetRID, failIfNotConfigured : true, cancellationToken : cancellationToken);

            return(await this.GetThroughputResponseAsync(
                       streamPayload : null,
                       operationType : OperationType.Read,
                       linkUri : new Uri(offerV2.SelfLink, UriKind.Relative),
                       resourceType : ResourceType.Offer,
                       requestOptions : requestOptions,
                       cancellationToken : cancellationToken));
        }
        internal async Task <OfferV2> GetOfferV2Async(
            string targetRID,
            CancellationToken cancellationToken = default(CancellationToken))
        {
            if (string.IsNullOrWhiteSpace(targetRID))
            {
                throw new ArgumentNullException(targetRID);
            }

            Offer offer = await this.ReadOfferAsync(targetRID, cancellationToken);

            OfferV2 offerV2 = offer as OfferV2;

            return(offerV2);
        }
        private async Task UpdateOfferThroughput(Connection connection, DocumentCollection collection, int throughtput)
        {
            var client = GetClient(connection);
            var query  = client.CreateOfferQuery().Where(o => o.ResourceLink == collection.SelfLink).AsDocumentQuery();

            var result = await query.ExecuteNextAsync <OfferV2>().ConfigureAwait(false);

            var offer = result.Single();

            if (offer.Content.OfferThroughput != throughtput)
            {
                offer = new OfferV2(offer, throughtput);
                await client.ReplaceOfferAsync(offer).ConfigureAwait(false);
            }
        }
        public async Task <ActionResponse> UpdateThroughput(string databaseId, string collectionId, int throughput)
        {
            var offer = await _cosmonautClient.GetOfferV2ForCollectionAsync(databaseId, collectionId);

            var newOffer = new OfferV2(offer, throughput);
            var result   = await _cosmonautClient.UpdateOfferAsync(newOffer);

            var success = result.StatusCode == HttpStatusCode.OK;

            return(new ActionResponse
            {
                Success = success,
                Message = success ? "Success" : "Failed"
            });
        }