[TestCategory("Quarantine")] // Not currently working with emulator
public async Task CreateDropAutoscaleContainerStreamApi()
{
DatabaseInternal database = (DatabaseInlineCore)await this.cosmosClient.CreateDatabaseAsync(
Guid.NewGuid().ToString());
ThroughputResponse databaseThroughput = await database.ReadThroughputIfExistsAsync(requestOptions : null);
Assert.IsNotNull(databaseThroughput);
Assert.AreEqual(HttpStatusCode.NotFound, databaseThroughput.StatusCode);
string streamContainerId = Guid.NewGuid().ToString();
using (ResponseMessage response = await database.CreateContainerStreamAsync(
new ContainerProperties(streamContainerId, "/pk"),
ThroughputProperties.CreateAutoscaleThroughput(5000)))
{
Assert.AreEqual(HttpStatusCode.Created, response.StatusCode);
ContainerInternal streamContainer = (ContainerInlineCore)database.GetContainer(streamContainerId);
ThroughputResponse autoscaleIfExists = await streamContainer.ReadThroughputIfExistsAsync(
requestOptions : null,
default(CancellationToken));
Assert.IsNotNull(autoscaleIfExists);
Assert.AreEqual(5000, autoscaleIfExists.Resource.AutoscaleMaxThroughput);
}
}
private static async Task <Container> CreateDemoEnvironment(CosmosClient client)
{
ContainerResponse hotelContainer = null;
try
{
// Set up a database
Microsoft.Azure.Cosmos.Database database = await client.CreateDatabaseIfNotExistsAsync(databaseId);
// Container and Throughput Properties
ContainerProperties containerProperties = new ContainerProperties(readHeavyId, "/CityName");
ThroughputProperties throughputProperties = ThroughputProperties.CreateAutoscaleThroughput(20000);
// Create a read heavy environment
hotelContainer = await database.CreateContainerAsync(
containerProperties,
throughputProperties);
}
catch (CosmosException ce)
{
Console.WriteLine($"Exception thrown by Cosmos DB: {ce.StatusCode}, {ce.Message}");
}
catch (Exception ex)
{
Console.WriteLine($"General Exception thrown: {ex.Message}");
throw;
}
return(hotelContainer);
}
[TestCategory("Quarantine")] // Not currently working with emulator
public async Task CreateDropAutoscaleContainer()
{
DatabaseInternal database = (DatabaseInlineCore)await this.cosmosClient.CreateDatabaseAsync(
Guid.NewGuid().ToString());
ContainerInternal container = (ContainerInlineCore)await database.CreateContainerAsync(
new ContainerProperties(Guid.NewGuid().ToString(), "/pk"),
ThroughputProperties.CreateAutoscaleThroughput(5000));
Assert.IsNotNull(container);
ThroughputResponse throughputResponse = await container.ReadThroughputAsync(requestOptions : null);
Assert.IsNotNull(throughputResponse);
Assert.AreEqual(5000, throughputResponse.Resource.AutoscaleMaxThroughput);
throughputResponse = await container.ReplaceThroughputAsync(
ThroughputProperties.CreateAutoscaleThroughput(6000),
requestOptions : null,
cancellationToken : default(CancellationToken));
Assert.IsNotNull(throughputResponse);
Assert.AreEqual(6000, throughputResponse.Resource.AutoscaleMaxThroughput);
await database.DeleteAsync();
}
public async Task CreateDropAutoscaleDatabaseStreamApi()
{
string databaseId = Guid.NewGuid().ToString();
using (ResponseMessage response = await this.cosmosClient.CreateDatabaseStreamAsync(
new DatabaseProperties(databaseId),
ThroughputProperties.CreateAutoscaleThroughput(5000)))
{
Assert.AreEqual(HttpStatusCode.Created, response.StatusCode);
}
DatabaseInternal database = (DatabaseInlineCore)this.cosmosClient.GetDatabase(databaseId);
ThroughputResponse autoscale = await database.ReadThroughputAsync(requestOptions : null);
Assert.IsNotNull(autoscale);
Assert.AreEqual(5000, autoscale.Resource.AutoscaleMaxThroughput);
ThroughputResponse autoscaleReplaced = await database.ReplaceThroughputAsync(
ThroughputProperties.CreateAutoscaleThroughput(10000));
Assert.IsNotNull(autoscaleReplaced);
Assert.AreEqual(10000, autoscaleReplaced.Resource.AutoscaleMaxThroughput);
await database.DeleteAsync();
}
public async Task CreateDatabaseIfNotExistTest()
{
string dbName = nameof(CreateDatabaseIfNotExistTest) + Guid.NewGuid();
DatabaseResponse databaseResponse = await this.cosmosClient.CreateDatabaseIfNotExistsAsync(
dbName,
ThroughputProperties.CreateAutoscaleThroughput(autoscaleMaxThroughput: 5000));
Assert.AreEqual(HttpStatusCode.Created, databaseResponse.StatusCode);
// Container is required to validate database throughput upgrade scenarios
Container container = await databaseResponse.Database.CreateContainerAsync("Test", "/id");
ThroughputResponse autoscale = await databaseResponse.Database.ReadThroughputAsync(requestOptions : null);
Assert.IsNotNull(autoscale);
Assert.IsNotNull(autoscale.Resource.Throughput);
Assert.AreEqual(5000, autoscale.Resource.AutoscaleMaxThroughput);
databaseResponse = await this.cosmosClient.CreateDatabaseIfNotExistsAsync(
dbName,
ThroughputProperties.CreateAutoscaleThroughput(
autoscaleMaxThroughput: 5000));
Assert.AreEqual(HttpStatusCode.OK, databaseResponse.StatusCode);
autoscale = await databaseResponse.Database.ReadThroughputAsync(requestOptions : null);
Assert.IsNotNull(autoscale);
Assert.IsNotNull(autoscale.Resource.Throughput);
Assert.AreEqual(5000, autoscale.Resource.AutoscaleMaxThroughput);
}
private void InitializeClient()
{
if (ConnectionString.IsNullOrEmpty())
{
ConnectionString = BuildStorageConnection().GetAwaiter().GetResult();
}
var clientBuilder = new CosmosClientBuilder(ConnectionString);
clientBuilder.WithThrottlingRetryOptions(TimeSpan.FromMilliseconds(500), 3);
var client = clientBuilder
.WithConnectionModeDirect()
.Build();
_cloudClient = client ?? throw new InvalidOperationException("Cannot build Cosmos Client using connection string");
// Create the database if it does not exist and been instructed to.
if (_createIfNotExists)
{
var throughputProperties = _autoscaleDatabaseThroughput
? ThroughputProperties.CreateAutoscaleThroughput(_maxThroughput)
: null;
_cloudClient.CreateDatabaseIfNotExistsAsync(DatabaseName, throughputProperties).GetAwaiter().GetResult();
}
// Create any tables required up front.
if (!_createTableNames.IsNullOrDefault())
{
foreach (var tableName in _createTableNames)
{
CreateTable(tableName).GetAwaiter().GetResult();
}
}
}
public async Task CreateDropAutoscaleAutoUpgradeDatabase()
{
DatabaseInternal database = (DatabaseInlineCore)await this.cosmosClient.CreateDatabaseAsync(
nameof(CreateDropAutoscaleAutoUpgradeDatabase) + Guid.NewGuid(),
ThroughputProperties.CreateAutoscaleThroughput(
maxAutoscaleThroughput: 5000,
autoUpgradeMaxThroughputIncrementPercentage: 10));
// Container is required to validate database throughput upgrade scenarios
Container container = await database.CreateContainerAsync("Test", "/id");
ThroughputResponse autoscale = await database.ReadThroughputAsync(requestOptions : null);
Assert.IsNotNull(autoscale);
Assert.AreEqual(5000, autoscale.Resource.AutoscaleMaxThroughput);
Assert.AreEqual(10, autoscale.Resource.AutoUpgradeMaxThroughputIncrementPercentage);
ThroughputResponse autoscaleReplaced = await database.ReplaceThroughputAsync(
ThroughputProperties.CreateAutoscaleThroughput(6000));
Assert.IsNotNull(autoscaleReplaced);
Assert.AreEqual(6000, autoscaleReplaced.Resource.AutoscaleMaxThroughput);
Assert.IsNull(autoscaleReplaced.Resource.AutoUpgradeMaxThroughputIncrementPercentage);
ThroughputResponse autoUpgradeReplace = await database.ReplaceThroughputAsync(
ThroughputProperties.CreateAutoscaleThroughput(
maxAutoscaleThroughput: 7000,
autoUpgradeMaxThroughputIncrementPercentage: 20));
Assert.IsNotNull(autoUpgradeReplace);
Assert.AreEqual(7000, autoUpgradeReplace.Resource.AutoscaleMaxThroughput);
Assert.AreEqual(20, autoUpgradeReplace.Resource.AutoUpgradeMaxThroughputIncrementPercentage);
await database.DeleteAsync();
}
private static async Task CreateImpl()
{
var cosmosClient = new CosmosClient("https://localhost:8081", "C2y6yDjf5/R+ob0N8A7Cgv30VRDJIWEHLM+4QDU5DE2nQ9nDuVTqobD4b8mGGyPMbIZnqyMsEcaGQy67XIw/Jw==");
var database = await cosmosClient.CreateDatabaseIfNotExistsAsync("db", ThroughputProperties.CreateAutoscaleThroughput(40000));
await database.Database.CreateContainerIfNotExistsAsync("graph", "/PartitionKey");
}
/// <summary>
/// Sets the provisioned throughput at database scope.
/// </summary>
/// <param name="model">The model.</param>
/// <param name="throughput">The throughput to set.</param>
/// <param name="autoscale">Whether autoscale is enabled.</param>
public static void SetThroughput(this IMutableModel model, int?throughput, bool?autoscale)
=> model.SetOrRemoveAnnotation(
CosmosAnnotationNames.Throughput,
throughput == null || autoscale == null
? null
: autoscale.Value
? ThroughputProperties.CreateAutoscaleThroughput(throughput.Value)
: ThroughputProperties.CreateManualThroughput(throughput.Value));
public async Task IngestDataNoSplit(Container container, int numPartitions = 3, int fieldLength = 100)
{
int numDocsPerPartition = 1000;
ThroughputProperties props = ThroughputProperties.CreateAutoscaleThroughput(10000);
await container.ReplaceThroughputAsync(props);
Func <string, int, int, Stream> func = (id, p, f) => { return(Utils.CreateRandomDocument(p, f, ref id)); };
await Utils.UploadRandomDataAsync(container, func, numPartitions, numDocsPerPartition, fieldLength, false);
}
public static async Task Main(string[] args)
{
Console.WriteLine($"Hello Cosmos DB World! {System.AppDomain.CurrentDomain.FriendlyName}");
var configuration = new ConfigurationBuilder()
.AddJsonFile("appsettings.json", optional: false, reloadOnChange: true)
.Build();
string endpoint = configuration["EndPointUrl"];
if (string.IsNullOrEmpty(endpoint))
{
throw new ArgumentNullException("Please specify a valid endpoint in the appSettings.json");
}
string authKey = configuration["AuthorizationKey"];
if (string.IsNullOrEmpty(authKey) || string.Equals(authKey, "Super secret key"))
{
throw new ArgumentException("Please specify a valid AuthorizationKey in the appSettings.json");
}
CosmosClientOptions options = new CosmosClientOptions()
{
AllowBulkExecution = true
};
client = new CosmosClient(endpoint, authKey, options);
DatabaseResponse databaseResponse = await client.CreateDatabaseIfNotExistsAsync(databaseName);
Database database = databaseResponse;
ContainerProperties containerProperties = new ContainerProperties(containerName, containerPartitionKey);
ContainerRequestOptions containerRequestOptions = new ContainerRequestOptions {
PopulateQuotaInfo = true
};
ContainerResponse containerResponse = await database.CreateContainerIfNotExistsAsync(
containerProperties : containerProperties,
throughputProperties : ThroughputProperties.CreateAutoscaleThroughput(autoscaleMaxThroughput: 4000),
containerRequestOptions
);
long indexTransformationProgress = long.Parse(containerResponse.Headers["x-ms-documentdb-collection-index-transformation-progress"]);
Console.WriteLine($" Index status {indexTransformationProgress}");
Container container = containerResponse.Container;
// await InsertSalesOrders(container, 1000000);
await QueryContainer(container);
Console.WriteLine("Done!");
}
public async Task IngestDataWithSplit(Container container, int numPartitions = 5, int fieldLength = 100)
{
Console.WriteLine("Starting IngestDataWithSplit.");
int numDocsPerPartition = 500;
Console.WriteLine("Ingesting data.");
await container.ReplaceThroughputAsync(ThroughputProperties.CreateAutoscaleThroughput(numPartitions * 10000));
await Utils.WaitForOfferReplaceAsync(container);
Func <string, int, int, Stream> func = (id, p, f) => { return(Utils.CreateRandomDocument(p, f, ref id)); };
await Utils.UploadRandomDataAsync(container, func, numPartitions, numDocsPerPartition, fieldLength, false);
}
public async Task IngestDataForDataTypes(Container container, int numPartitions = 5)
{
int numDocsPerPartition = 1000;
Func <string, int, int, Stream> func = (id, p, f) =>
{
DataTypeDocument doc = DataTypeDocument.CreateRandomDocument(id, p, f);
return(doc.SerializeToStream());
};
await container.ReplaceThroughputAsync(ThroughputProperties.CreateAutoscaleThroughput(10000));
await Utils.UploadRandomDataAsync(container, func, numPartitions, numDocsPerPartition, 100, true);
}
public async Task <int> UpdateDatabaseThroughput(string name, int sharedRu)
{
try {
Database db = client.GetDatabase(name);
ThroughputProperties props = ThroughputProperties.CreateAutoscaleThroughput(sharedRu);
ThroughputResponse resp = await db.ReplaceThroughputAsync(props);
return((int)resp.StatusCode);
}
catch (Exception e) {
Console.WriteLine($"UpdateDatabaseThroughput {name} {sharedRu} -> Exception {e}");
return(-1);
}
}
/// <summary>
/// Sets the provisioned throughput at database scope.
/// </summary>
/// <param name="model">The model.</param>
/// <param name="throughput">The throughput to set.</param>
/// <param name="autoscale">Whether autoscale is enabled.</param>
/// <param name="fromDataAnnotation">Indicates whether the configuration was specified using a data annotation.</param>
public static int?SetThroughput(
this IConventionModel model,
int?throughput,
bool?autoscale,
bool fromDataAnnotation = false)
{
var valueSet = (ThroughputProperties?)model.SetOrRemoveAnnotation(
CosmosAnnotationNames.Throughput,
throughput == null || autoscale == null
? null
: autoscale.Value
? ThroughputProperties.CreateAutoscaleThroughput(throughput.Value)
: ThroughputProperties.CreateManualThroughput(throughput.Value),
fromDataAnnotation)?.Value;
return(valueSet?.AutoscaleMaxThroughput ?? valueSet?.Throughput);
}
private static async Task SetContainerPerformance(Container container, int desiredThroughput)
{
ThroughputProperties throughputProperties = await container.ReadThroughputAsync(requestOptions : null);
ThroughputResponse throughputResponse = null;
if (throughputProperties.AutoscaleMaxThroughput != null)
{
// Container configured with autoscale throughput
if (throughputProperties.AutoscaleMaxThroughput.Value == desiredThroughput)
{
WriteLineInColor($"\nThe {container.Id} container is already configured with max throughput: {desiredThroughput}. Skipping performance change request...", ConsoleColor.Yellow);
}
else
{
WriteLineInColor($"\nThe {container.Id} container is configured with max throughput: {throughputProperties.AutoscaleMaxThroughput}\nChanging max throughput to {desiredThroughput}", ConsoleColor.White);
ThroughputProperties newThroughputProperties = ThroughputProperties.CreateAutoscaleThroughput(desiredThroughput);
throughputResponse = await container.ReplaceThroughputAsync(newThroughputProperties);
WriteLineInColor($"\nChanged {container.Id}'s max throughput to {desiredThroughput}", ConsoleColor.Cyan);
}
}
else
{
// Container configured with manual throughput
if (throughputProperties.Throughput.HasValue && throughputProperties.Throughput.Value == desiredThroughput)
{
WriteLineInColor($"\nThe {container.Id} container is already configured with throughput: {desiredThroughput}. Skipping performance change request...", ConsoleColor.Yellow);
}
else
{
WriteLineInColor($"\nThe {container.Id} container is configured with throughput: {throughputProperties.AutoscaleMaxThroughput}\nChanging throughput to {desiredThroughput}", ConsoleColor.White);
ThroughputProperties newThroughputProperties = ThroughputProperties.CreateManualThroughput(desiredThroughput);
throughputResponse = await container.ReplaceThroughputAsync(newThroughputProperties);
WriteLineInColor($"\nChanged {container.Id}'s throughput to {desiredThroughput}", ConsoleColor.Cyan);
}
}
}
[TestCategory("Quarantine")] // Not currently working with emulator
public async Task ReadFixedWithAutoscaleTests()
{
DatabaseInternal database = (DatabaseInlineCore)await this.cosmosClient.CreateDatabaseAsync(
Guid.NewGuid().ToString());
ContainerInternal autoscaleContainer = (ContainerInlineCore)await database.CreateContainerAsync(
new ContainerProperties(Guid.NewGuid().ToString(), "/pk"),
ThroughputProperties.CreateAutoscaleThroughput(5000));
Assert.IsNotNull(autoscaleContainer);
// Reading a autoscale container with fixed results
int?throughput = await autoscaleContainer.ReadThroughputAsync();
Assert.IsNotNull(throughput);
await database.DeleteAsync();
}
// </CreateContainer>
// <CreateAndUpdateAutoscaleContainer>
private static async Task CreateAndUpdateAutoscaleContainer()
{
// Set autoscale throughput to the maximum value of 10000 RU/s
ContainerProperties containerProperties = new ContainerProperties(autoscaleContainerId, partitionKeyPath);
Container autoscaleContainer = await database.CreateContainerIfNotExistsAsync(
containerProperties : containerProperties,
throughputProperties : ThroughputProperties.CreateAutoscaleThroughput(autoscaleMaxThroughput: 10000));
Console.WriteLine($"{Environment.NewLine}1.2. Created autoscale container :{autoscaleContainer.Id}");
//*********************************************************************************************
// Get configured performance of a CosmosContainer
//**********************************************************************************************
ThroughputResponse throughputResponse = await autoscaleContainer.ReadThroughputAsync(requestOptions : null);
Console.WriteLine($"{Environment.NewLine}1.2.1. Found autoscale throughput {Environment.NewLine}The current throughput: {throughputResponse.Resource.Throughput} Max throughput: {throughputResponse.Resource.AutoscaleMaxThroughput} " +
$"using container's id: {autoscaleContainer.Id}");
//*********************************************************************************************
// Get the current throughput configured for a Container
//**********************************************************************************************
int?currentThroughput = await autoscaleContainer.ReadThroughputAsync();
Console.WriteLine($"{Environment.NewLine}1.2.2. Found autoscale throughput {Environment.NewLine}The current throughput: {currentThroughput} using container's id: {autoscaleContainer.Id}");
//******************************************************************************************************************
// Change performance (reserved throughput) of CosmosContainer
// Let's change the performance of the autoscale container to a maximum throughput of 15000 RU/s
//******************************************************************************************************************
ThroughputResponse throughputUpdateResponse = await autoscaleContainer.ReplaceThroughputAsync(ThroughputProperties.CreateAutoscaleThroughput(15000));
Console.WriteLine($"{Environment.NewLine}1.2.3. Replaced autoscale throughput. {Environment.NewLine}The current throughput: {throughputUpdateResponse.Resource.Throughput} Max throughput: {throughputUpdateResponse.Resource.AutoscaleMaxThroughput} " +
$"using container's id: {autoscaleContainer.Id}");
// Get the offer again after replace
throughputResponse = await autoscaleContainer.ReadThroughputAsync(requestOptions : null);
Console.WriteLine($"{Environment.NewLine}1.2.4. Found autoscale throughput {Environment.NewLine}The current throughput: {throughputResponse.Resource.Throughput} Max throughput: {throughputResponse.Resource.AutoscaleMaxThroughput} " +
$"using container's id: {autoscaleContainer.Id}{Environment.NewLine}");
// Delete the container
await autoscaleContainer.DeleteContainerAsync();
}
[TestCategory("Quarantine")] // Not currently working with emulator
public async Task ContainerAutoscaleIfExistsTest()
{
DatabaseInternal database = (DatabaseInlineCore)await this.cosmosClient.CreateDatabaseAsync(
nameof(CreateDropAutoscaleDatabase) + Guid.NewGuid().ToString());
Container container = await database.CreateContainerAsync(
containerProperties : new ContainerProperties("Test", "/id"),
throughputProperties : ThroughputProperties.CreateAutoscaleThroughput(5000));
ContainerInternal containerCore = (ContainerInlineCore)container;
ThroughputResponse throughputResponse = await database.ReadThroughputIfExistsAsync(requestOptions : null);
Assert.IsNotNull(throughputResponse);
Assert.AreEqual(HttpStatusCode.NotFound, throughputResponse.StatusCode);
Assert.IsNull(throughputResponse.Resource);
throughputResponse = await database.ReplaceThroughputPropertiesIfExistsAsync(
ThroughputProperties.CreateAutoscaleThroughput(6000));
Assert.IsNotNull(throughputResponse);
Assert.AreEqual(HttpStatusCode.NotFound, throughputResponse.StatusCode);
Assert.IsNull(throughputResponse.Resource);
throughputResponse = await containerCore.ReadThroughputIfExistsAsync(
requestOptions : null,
default(CancellationToken));
Assert.IsNotNull(throughputResponse);
Assert.IsTrue(throughputResponse.Resource.Throughput > 400);
Assert.AreEqual(5000, throughputResponse.Resource.AutoscaleMaxThroughput);
throughputResponse = await containerCore.ReplaceThroughputIfExistsAsync(
ThroughputProperties.CreateAutoscaleThroughput(6000),
requestOptions : null,
default(CancellationToken));
Assert.IsNotNull(throughputResponse);
Assert.IsTrue(throughputResponse.Resource.Throughput > 400);
Assert.AreEqual(6000, throughputResponse.Resource.AutoscaleMaxThroughput);
await database.DeleteAsync();
}
public async Task CreateDropAutoscaleDatabase()
{
DatabaseInternal database = (DatabaseInlineCore)await this.cosmosClient.CreateDatabaseAsync(
nameof(CreateDropAutoscaleDatabase) + Guid.NewGuid().ToString(),
ThroughputProperties.CreateAutoscaleThroughput(5000));
ThroughputResponse autoscale = await database.ReadThroughputAsync(requestOptions : null);
Assert.IsNotNull(autoscale);
Assert.AreEqual(5000, autoscale.Resource.AutoscaleMaxThroughput);
ThroughputResponse autoscaleReplaced = await database.ReplaceThroughputAsync(
ThroughputProperties.CreateAutoscaleThroughput(10000));
Assert.IsNotNull(autoscaleReplaced);
Assert.AreEqual(10000, autoscaleReplaced.Resource.AutoscaleMaxThroughput);
await database.DeleteAsync();
}
public async Task <DatabaseResponse> CreateDatabase(string name, int sharedRu)
{
if (sharedRu > 0)
{
if (sharedRu < 4000) // min sharedRu value is 4000
{
ThroughputProperties props = ThroughputProperties.CreateAutoscaleThroughput(4000);
return(await client.CreateDatabaseIfNotExistsAsync(name, throughputProperties : props));
}
else
{
ThroughputProperties props = ThroughputProperties.CreateAutoscaleThroughput(sharedRu);
return(await client.CreateDatabaseIfNotExistsAsync(name, throughputProperties : props));
}
}
else
{
return(await client.CreateDatabaseIfNotExistsAsync(name));
}
}
private static async Task InitializeCosmosDb()
{
Database database = await _cosmosDbClient.CreateDatabaseIfNotExistsAsync(DatabaseName);
// Create a partition based on the ipCountryCode value from the Transactions data set.
// This partition was selected because the data will most likely include this value, and
// it allows us to partition by location from which the transaction originated. This field
// also contains a wide range of values, which is preferable for partitions.
var containerProperties = new ContainerProperties(CollectionName, PartitionKey)
{
// The default indexing policy for newly created containers is set to automatic and enforces
// range indexes for any string or number.
IndexingPolicy = { Automatic = true }
};
// Create the container with a maximum autoscale throughput of 4000 RU/s. By default, the
// minimum RU/s will be 400.
var container = await database.CreateContainerIfNotExistsAsync(
containerProperties : containerProperties,
throughputProperties : ThroughputProperties.CreateAutoscaleThroughput(autoscaleMaxThroughput: 4000));
}
private ThroughputProperties GetThroughputProperties()
{
string throughput;
ThroughputProperties properties = null;
if (!string.IsNullOrEmpty(this.Configuration.GetValueOrDefault("AutoscaleThroughput")))
{
throughput = this.Configuration.GetValueOrDefault("AutoscaleThroughput");
bool valid = int.TryParse(throughput, out int autoscaleThroughput);
if (valid)
{
properties = ThroughputProperties.CreateAutoscaleThroughput(autoscaleThroughput);
return(properties);
}
}
if (!string.IsNullOrEmpty(this.Configuration.GetValueOrDefault("ManualThroughput")))
{
throughput = this.Configuration.GetValueOrDefault("ManualThroughput");
bool valid = int.TryParse(throughput, out int manualThroughput);
if (valid)
{
properties = ThroughputProperties.CreateManualThroughput(manualThroughput);
return(properties);
}
}
properties = ThroughputProperties.CreateAutoscaleThroughput(4000);
return(properties);
}
public async Task DatabaseAutoscaleIfExistsTest()
{
DatabaseInternal database = (DatabaseInlineCore)await this.cosmosClient.CreateDatabaseAsync(
nameof(CreateDropAutoscaleDatabase) + Guid.NewGuid().ToString(),
ThroughputProperties.CreateAutoscaleThroughput(5000));
Container container = await database.CreateContainerAsync("Test", "/id");
ContainerInternal containerCore = (ContainerInlineCore)container;
ThroughputResponse throughputResponse = await database.ReadThroughputIfExistsAsync(requestOptions : null);
Assert.IsNotNull(throughputResponse);
Assert.AreEqual(5000, throughputResponse.Resource.AutoscaleMaxThroughput);
throughputResponse = await database.ReplaceThroughputAsync(
ThroughputProperties.CreateAutoscaleThroughput(6000));
Assert.IsNotNull(throughputResponse);
Assert.AreEqual(6000, throughputResponse.Resource.AutoscaleMaxThroughput);
throughputResponse = await containerCore.ReadThroughputIfExistsAsync(
requestOptions : null,
public async Task CreateContainerIfNotExistTest()
{
string dbName = nameof(CreateContainerIfNotExistTest) + Guid.NewGuid();
DatabaseCore database = (DatabaseInlineCore)await this.cosmosClient.CreateDatabaseAsync(dbName);
ContainerProperties containerProperties = new ContainerProperties("Test", "/id");
ContainerResponse containerResponse = await database.CreateContainerIfNotExistsAsync(
containerProperties,
ThroughputProperties.CreateAutoscaleThroughput(5000));
Assert.AreEqual(HttpStatusCode.Created, containerResponse.StatusCode);
ThroughputResponse autoscale = await containerResponse.Container.ReadThroughputAsync(requestOptions : null);
Assert.IsNotNull(autoscale);
Assert.IsNotNull(autoscale.Resource.Throughput);
Assert.AreEqual(5000, autoscale.Resource.AutoscaleMaxThroughput);
containerResponse = await database.CreateContainerIfNotExistsAsync(
containerProperties,
ThroughputProperties.CreateAutoscaleThroughput(5000));
Assert.AreEqual(HttpStatusCode.OK, containerResponse.StatusCode);
}
/// <summary>
/// Creates a new database
/// </summary>
private async Task CreateDatabaseIfNotExistsAsync()
{
var throughputProperties = ThroughputProperties.CreateAutoscaleThroughput(MaxAutoScaleThroughput);
await cosmosClient.CreateDatabaseIfNotExistsAsync(container.Database.Id, throughputProperties);
}
public async Task AutoscaleThroughputSerializationTest()
{
ThroughputProperties autoscaleThroughputProperties = ThroughputProperties.CreateAutoscaleThroughput(1000);
Assert.AreEqual(1000, autoscaleThroughputProperties.AutoscaleMaxThroughput);
Assert.IsNull(autoscaleThroughputProperties.Throughput);
using (Stream stream = MockCosmosUtil.Serializer.ToStream <ThroughputProperties>(autoscaleThroughputProperties))
{
using (StreamReader reader = new StreamReader(stream))
{
string output = await reader.ReadToEndAsync();
Assert.IsNotNull(output);
Assert.AreEqual("{\"content\":{\"offerAutopilotSettings\":{\"maxThroughput\":1000}},\"offerVersion\":\"V2\"}", output);
}
}
OfferAutoscaleProperties autoscaleProperties = new OfferAutoscaleProperties(
startingMaxThroughput: 1000,
autoUpgradeMaxThroughputIncrementPercentage: null);
using (Stream stream = MockCosmosUtil.Serializer.ToStream <OfferAutoscaleProperties>(autoscaleProperties))
{
using (StreamReader reader = new StreamReader(stream))
{
string output = await reader.ReadToEndAsync();
Assert.IsNotNull(output);
Assert.AreEqual("{\"maxThroughput\":1000}", output);
}
}
using (Stream stream = MockCosmosUtil.Serializer.ToStream <OfferAutoscaleProperties>(autoscaleProperties))
{
OfferAutoscaleProperties fromStream = MockCosmosUtil.Serializer.FromStream <OfferAutoscaleProperties>(stream);
Assert.IsNotNull(fromStream);
Assert.AreEqual(1000, fromStream.MaxThroughput);
}
OfferContentProperties content = OfferContentProperties.CreateAutoscaleOfferConent(
startingMaxThroughput: 1000,
autoUpgradeMaxThroughputIncrementPercentage: null);
using (Stream stream = MockCosmosUtil.Serializer.ToStream <OfferContentProperties>(content))
{
using (StreamReader reader = new StreamReader(stream))
{
string output = await reader.ReadToEndAsync();
Assert.IsNotNull(output);
Assert.AreEqual("{\"offerAutopilotSettings\":{\"maxThroughput\":1000}}", output);
}
}
using (Stream stream = MockCosmosUtil.Serializer.ToStream <OfferContentProperties>(content))
{
OfferContentProperties fromStream = MockCosmosUtil.Serializer.FromStream <OfferContentProperties>(stream);
Assert.IsNotNull(fromStream.OfferAutoscaleSettings);
Assert.AreEqual(1000, fromStream.OfferAutoscaleSettings.MaxThroughput);
Assert.IsNull(fromStream.OfferThroughput);
}
using (Stream stream = MockCosmosUtil.Serializer.ToStream <ThroughputProperties>(autoscaleThroughputProperties))
{
ThroughputProperties fromStream = MockCosmosUtil.Serializer.FromStream <ThroughputProperties>(stream);
Assert.AreEqual(1000, fromStream.AutoscaleMaxThroughput);
Assert.IsNull(fromStream.Throughput);;
}
}
private static async Task InitializeCosmosDb()
{
_database = await _cosmosDbClient.CreateDatabaseIfNotExistsAsync(DatabaseName);
#region Telemetry container
// Create telemetry container with analytical store enabled
var telemetryContainerDefinition =
new ContainerProperties(id: TelemetryContainerName, partitionKeyPath: $"/{PartitionKey}")
{
IndexingPolicy = { IndexingMode = IndexingMode.Consistent },
AnalyticalStoreTimeToLiveInSeconds = -1
};
// Tune the indexing policy for write-heavy workloads by only including regularly queried paths.
// Be careful when using an opt-in policy as we are below. Excluding all and only including certain paths removes
// Cosmos DB's ability to proactively add new properties to the index.
telemetryContainerDefinition.IndexingPolicy.ExcludedPaths.Clear();
telemetryContainerDefinition.IndexingPolicy.ExcludedPaths.Add(new ExcludedPath {
Path = "/*"
}); // Exclude all paths.
telemetryContainerDefinition.IndexingPolicy.IncludedPaths.Clear();
telemetryContainerDefinition.IndexingPolicy.IncludedPaths.Add(new IncludedPath {
Path = "/vin/?"
});
telemetryContainerDefinition.IndexingPolicy.IncludedPaths.Add(new IncludedPath {
Path = "/state/?"
});
telemetryContainerDefinition.IndexingPolicy.IncludedPaths.Add(new IncludedPath {
Path = "/partitionKey/?"
});
// Provision the container with autoscale throughput
ThroughputProperties telemetryThroughputProperties = ThroughputProperties.CreateAutoscaleThroughput(TELEMETRY_CONTAINER_RUS);
// Create the container with autoscale throughput
var telemetryContainerResponse = await _database.CreateContainerIfNotExistsAsync(telemetryContainerDefinition, telemetryThroughputProperties);
#endregion
#region Metadata container
// Create metadata container with analytical store enabled
var metadataContainerDefinition =
new ContainerProperties(id: MetadataContainerName, partitionKeyPath: $"/{PartitionKey}")
{
// Set the indexing policy to consistent and use the default settings because we expect read-heavy workloads in this container (includes all paths (/*) with all range indexes).
// Indexing all paths when you have write-heavy workloads may impact performance and cost more RU/s than desired.
IndexingPolicy = { IndexingMode = IndexingMode.Consistent },
AnalyticalStoreTimeToLiveInSeconds = -1
};
// Provision the container with autoscale throughput - start with bulk import throughput since the bulk import will run after this create
ThroughputProperties metadataThroughputProperties = ThroughputProperties.CreateAutoscaleThroughput(METADATA_CONTAINER_RUS_BULK_IMPORT);
// Create the container with autoscale throughput
var metadataContainerResponse = await _database.CreateContainerIfNotExistsAsync(metadataContainerDefinition, metadataThroughputProperties);
#endregion
#region Maintenance container
// Create maintenance container
var maintenanceContainerDefinition =
new ContainerProperties(id: MaintenanceContainerName, partitionKeyPath: $"/vin")
{
IndexingPolicy = { IndexingMode = IndexingMode.Consistent }
};
// Provision the container with fixed throughput
ThroughputProperties maintenanceThroughputProperties = ThroughputProperties.CreateManualThroughput(MAINTENANCE_CONTAINER_RUS);
// Create the container
var maintenanceContainerResponse = await _database.CreateContainerIfNotExistsAsync(maintenanceContainerDefinition, maintenanceThroughputProperties);
#endregion
}
private static async Task ImportToCosmosDb(IConfiguration configuration)
{
var total = Files.Count;
var failures = 0;
var endpointUrl = configuration["CosmosDb:EndpointUrl"];
var authorizationKey = configuration["CosmosDb:AuthorizationKey"];
var databaseName = configuration["CosmosDb:DatabaseName"];
var containerName = configuration["CosmosDb:ContainerName"];
var client = new CosmosClient(endpointUrl, authorizationKey, new CosmosClientOptions()
{
AllowBulkExecution = true
});
Database database = await client.CreateDatabaseIfNotExistsAsync(databaseName);
var containerProperties = new ContainerProperties(containerName, "/fullname");
var throughputProperties = ThroughputProperties.CreateAutoscaleThroughput(5000);
Container container = await database.CreateContainerIfNotExistsAsync(containerProperties, throughputProperties);
Console.WriteLine($"Got {total} files, processing...");
var itemsToInsert = new Dictionary <PartitionKey, Stream>(total);
while (!Files.IsEmpty && Files.TryDequeue(out var file))
{
var stream = new MemoryStream();
await JsonSerializer.SerializeAsync(stream, file);
itemsToInsert.Add(new PartitionKey(file.fullname), stream);
}
Console.WriteLine("Importing to database, please wait patiently...");
var stopwatch = Stopwatch.StartNew();
List <Task> tasks = new List <Task>(total);
foreach (KeyValuePair <PartitionKey, Stream> item in itemsToInsert)
{
tasks.Add(CreateOrUpdateAsync(container, item.Value, item.Key)
.ContinueWith((Task <ResponseMessage> task) =>
{
using ResponseMessage response = task.Result;
if (!response.IsSuccessStatusCode)
{
Interlocked.Increment(ref failures);
Console.WriteLine($"Received {response.StatusCode} ({response.ErrorMessage}).");
}
}));
}
//Interlocked.Increment();
// Wait until all are done
await Task.WhenAll(tasks);
stopwatch.Stop();
Console.WriteLine();
Console.WriteLine("===========================================");
Console.WriteLine($"Import data completed：");
Console.WriteLine($"- Total：{total}");
Console.WriteLine($"- Succeed：{total - failures}");
Console.WriteLine($"- Failed：{failures}");
Console.WriteLine($"- Elapsed：{stopwatch.Elapsed}");
Console.WriteLine("===========================================");
Console.WriteLine();
}
public async Task ContainerBuilderAutoscaleTest()
{
DatabaseInternal database = (DatabaseInlineCore)await this.cosmosClient.CreateDatabaseAsync(
nameof(CreateDropAutoscaleDatabase) + Guid.NewGuid().ToString());
{
Container container = await database.DefineContainer("Test", "/id")
.CreateAsync(throughputProperties: ThroughputProperties.CreateAutoscaleThroughput(5000));
ThroughputProperties throughputProperties = await container.ReadThroughputAsync(requestOptions : null);
Assert.IsNotNull(throughputProperties);
Assert.IsTrue(throughputProperties.Throughput > 400);
Assert.AreEqual(5000, throughputProperties.AutoscaleMaxThroughput);
}
{
Container container2 = await database.DefineContainer("Test2", "/id")
.CreateIfNotExistsAsync(throughputProperties: ThroughputProperties.CreateAutoscaleThroughput(5000));
ThroughputProperties throughputProperties = await container2.ReadThroughputAsync(requestOptions : null);
Assert.IsNotNull(throughputProperties);
Assert.IsTrue(throughputProperties.Throughput > 400);
Assert.AreEqual(5000, throughputProperties.AutoscaleMaxThroughput);
container2 = await database.DefineContainer(container2.Id, "/id")
.CreateIfNotExistsAsync(throughputProperties: ThroughputProperties.CreateAutoscaleThroughput(5000));
throughputProperties = await container2.ReadThroughputAsync(requestOptions : null);
Assert.IsNotNull(throughputProperties);
Assert.IsTrue(throughputProperties.Throughput > 400);
Assert.AreEqual(5000, throughputProperties.AutoscaleMaxThroughput);
}
{
Container container3 = await database.DefineContainer("Test3", "/id")
.CreateAsync(throughputProperties: ThroughputProperties.CreateManualThroughput(500));
ThroughputProperties throughputProperties = await container3.ReadThroughputAsync(requestOptions : null);
Assert.IsNotNull(throughputProperties);
Assert.IsNull(throughputProperties.AutoscaleMaxThroughput);
Assert.AreEqual(500, throughputProperties.Throughput);
container3 = await database.DefineContainer(container3.Id, "/id")
.CreateIfNotExistsAsync(throughputProperties: ThroughputProperties.CreateManualThroughput(500));
throughputProperties = await container3.ReadThroughputAsync(requestOptions : null);
Assert.IsNotNull(throughputProperties);
Assert.IsNull(throughputProperties.AutoscaleMaxThroughput);
Assert.AreEqual(500, throughputProperties.Throughput);
}
{
Container container4 = await database.DefineContainer("Test4", "/id")
.CreateIfNotExistsAsync(throughputProperties: ThroughputProperties.CreateManualThroughput(500));
ThroughputProperties throughputProperties = await container4.ReadThroughputAsync(requestOptions : null);
Assert.IsNotNull(throughputProperties);
Assert.IsNull(throughputProperties.AutoscaleMaxThroughput);
Assert.AreEqual(500, throughputProperties.Throughput);
}
await database.DeleteAsync();
}