public async Task IngestDataWithSplit(Container container, int numPartitions = 5, int fieldLength = 100)
{
Console.WriteLine("Starting IngestDataWithSplit.");
int numDocsPerPartition = 500;
Console.WriteLine("Ingesting data.");
await container.ReplaceThroughputAsync(ThroughputProperties.CreateAutoscaleThroughput(numPartitions * 10000));
await Utils.WaitForOfferReplaceAsync(container);
Func <string, int, int, Stream> func = (id, p, f) => { return(Utils.CreateRandomDocument(p, f, ref id)); };
await Utils.UploadRandomDataAsync(container, func, numPartitions, numDocsPerPartition, fieldLength, false);
}
public async Task IngestDataForDataTypes(Container container, int numPartitions = 5)
{
int numDocsPerPartition = 1000;
Func <string, int, int, Stream> func = (id, p, f) =>
{
DataTypeDocument doc = DataTypeDocument.CreateRandomDocument(id, p, f);
return(doc.SerializeToStream());
};
await container.ReplaceThroughputAsync(ThroughputProperties.CreateAutoscaleThroughput(10000));
await Utils.UploadRandomDataAsync(container, func, numPartitions, numDocsPerPartition, 100, true);
}
public async Task ReadReplaceThroughputResponseTests()
{
string databaseId = Guid.NewGuid().ToString();
int throughput = 10000;
DatabaseResponse createResponse = await this.CreateDatabaseHelper(databaseId, databaseExists : false, throughput : throughput);
Assert.AreEqual(HttpStatusCode.Created, createResponse.StatusCode);
Cosmos.Database cosmosDatabase = createResponse;
ThroughputResponse readThroughputResponse = await cosmosDatabase.ReadThroughputAsync(new RequestOptions());
Assert.IsNotNull(readThroughputResponse);
Assert.IsNotNull(readThroughputResponse.Resource);
Assert.IsNotNull(readThroughputResponse.MinThroughput);
Assert.IsNotNull(readThroughputResponse.Resource.Throughput);
Assert.AreEqual(throughput, readThroughputResponse.Resource.Throughput.Value);
// Implicit
ThroughputProperties throughputProperties = await cosmosDatabase.ReadThroughputAsync(new RequestOptions());
Assert.IsNotNull(throughputProperties);
Assert.AreEqual(throughput, throughputProperties.Throughput);
// Simple API
int?readThroughput = await cosmosDatabase.ReadThroughputAsync();
Assert.AreEqual(throughput, readThroughput);
string containerId = Guid.NewGuid().ToString();
string partitionPath = "/users";
ContainerResponse containerResponse = await cosmosDatabase.CreateContainerAsync(containerId, partitionPath);
Assert.AreEqual(HttpStatusCode.Created, containerResponse.StatusCode);
ThroughputResponse replaceThroughputResponse = await cosmosDatabase.ReplaceThroughputAsync(readThroughputResponse.Resource.Throughput.Value + 1000);
Assert.IsNotNull(replaceThroughputResponse);
Assert.IsNotNull(replaceThroughputResponse.Resource);
Assert.AreEqual(readThroughputResponse.Resource.Throughput.Value + 1000, replaceThroughputResponse.Resource.Throughput.Value);
Container container = containerResponse;
readThroughputResponse = await container.ReadThroughputAsync(new RequestOptions());
Assert.IsNull(readThroughputResponse.Resource);
await container.DeleteContainerAsync();
await cosmosDatabase.DeleteAsync();
}
public async Task <int> UpdateDatabaseThroughput(string name, int sharedRu)
{
try {
Database db = client.GetDatabase(name);
ThroughputProperties props = ThroughputProperties.CreateAutoscaleThroughput(sharedRu);
ThroughputResponse resp = await db.ReplaceThroughputAsync(props);
return((int)resp.StatusCode);
}
catch (Exception e) {
Console.WriteLine($"UpdateDatabaseThroughput {name} {sharedRu} -> Exception {e}");
return(-1);
}
}
public async Task <Container> InitializeCollection(CosmosClient client)
{
Database database = client.GetDatabase(_cosmosDataStoreConfiguration.DatabaseId);
Container containerClient = database.GetContainer(_cosmosCollectionConfiguration.CollectionId);
_logger.LogInformation("Finding Container: {collectionId}", _cosmosCollectionConfiguration.CollectionId);
AsyncPolicy retryPolicy = _retryExceptionPolicyFactory.GetRetryPolicy();
var existingContainer = await retryPolicy.ExecuteAsync(async() => await database.TryGetContainerAsync(_cosmosCollectionConfiguration.CollectionId));
_logger.LogInformation("Creating Cosmos Container if not exits: {collectionId}", _cosmosCollectionConfiguration.CollectionId);
ContainerResponse containerResponse = await retryPolicy.ExecuteAsync(async() =>
await database.CreateContainerIfNotExistsAsync(
_cosmosCollectionConfiguration.CollectionId,
$"/{KnownDocumentProperties.PartitionKey}",
_cosmosCollectionConfiguration.InitialCollectionThroughput));
if (containerResponse.StatusCode == HttpStatusCode.Created || containerResponse.Resource.DefaultTimeToLive != -1)
{
if (_cosmosCollectionConfiguration.InitialCollectionThroughput.HasValue)
{
var throughputProperties = ThroughputProperties.CreateManualThroughput(_cosmosCollectionConfiguration.InitialCollectionThroughput.Value);
await retryPolicy.ExecuteAsync(async() => await containerClient.ReplaceThroughputAsync(throughputProperties));
}
containerResponse.Resource.DefaultTimeToLive = -1;
existingContainer = await retryPolicy.ExecuteAsync(async() => await containerClient.ReplaceContainerAsync(containerResponse));
}
await retryPolicy.ExecuteAsync(async() =>
{
try
{
await _clientTestProvider.PerformTest(existingContainer, _cosmosDataStoreConfiguration, _cosmosCollectionConfiguration);
}
catch (CosmosException e) when(e.StatusCode == HttpStatusCode.TooManyRequests)
{
// This is the very first interaction with the collection, and we might get this exception
// when it calls GetCachedContainerPropertiesAsync, which does not use our request handler.
throw new RequestRateExceededException(e.RetryAfter);
}
});
await _upgradeManager.SetupContainerAsync(containerClient);
return(existingContainer);
}
private async Task CreateContainerIfNotExistsAsync()
{
this.container = $"{typeof(TEntity).Name}Container";
ContainerProperties properties = new(id : this.container, partitionKeyPath : this.GetPartitionKey());
ThroughputProperties throughputProperties = this.GetThroughputProperties();
ContainerResponse response = await this.Database.CreateContainerIfNotExistsAsync(properties, throughputProperties).ConfigureAwait(false);
if ((response.StatusCode != HttpStatusCode.OK && response.StatusCode != HttpStatusCode.Created) || response.Container == null)
{
throw new PrismaException(PrismaError.ErrorInitializeProvider, Properties.Resources.RES_Exception_Initializing_Resource.Format(nameof(Container)));
}
}
public override async Task <DatabaseResponse> CreateDatabaseAsync(
string id,
ThroughputProperties throughputProperties,
RequestOptions requestOptions = null,
CancellationToken cancellationToken = default)
{
EncryptionDatabaseResponse encryptionDatabaseResponse = new EncryptionDatabaseResponse(
await this.cosmosClient.CreateDatabaseAsync(
id,
throughputProperties,
requestOptions,
cancellationToken),
this);
return(encryptionDatabaseResponse);
}
/// <summary>
/// Sets the provisioned throughput at database scope.
/// </summary>
/// <param name="model">The model.</param>
/// <param name="throughput">The throughput to set.</param>
/// <param name="autoscale">Whether autoscale is enabled.</param>
/// <param name="fromDataAnnotation">Indicates whether the configuration was specified using a data annotation.</param>
public static int?SetThroughput(
this IConventionModel model,
int?throughput,
bool?autoscale,
bool fromDataAnnotation = false)
{
var valueSet = (ThroughputProperties?)model.SetOrRemoveAnnotation(
CosmosAnnotationNames.Throughput,
throughput == null || autoscale == null
? null
: autoscale.Value
? ThroughputProperties.CreateAutoscaleThroughput(throughput.Value)
: ThroughputProperties.CreateManualThroughput(throughput.Value),
fromDataAnnotation)?.Value;
return(valueSet?.AutoscaleMaxThroughput ?? valueSet?.Throughput);
}
private static async Task SetContainerPerformance(Container container, int desiredThroughput)
{
ThroughputProperties throughputProperties = await container.ReadThroughputAsync(requestOptions : null);
ThroughputResponse throughputResponse = null;
if (throughputProperties.AutoscaleMaxThroughput != null)
{
// Container configured with autoscale throughput
if (throughputProperties.AutoscaleMaxThroughput.Value == desiredThroughput)
{
WriteLineInColor($"\nThe {container.Id} container is already configured with max throughput: {desiredThroughput}. Skipping performance change request...", ConsoleColor.Yellow);
}
else
{
WriteLineInColor($"\nThe {container.Id} container is configured with max throughput: {throughputProperties.AutoscaleMaxThroughput}\nChanging max throughput to {desiredThroughput}", ConsoleColor.White);
ThroughputProperties newThroughputProperties = ThroughputProperties.CreateAutoscaleThroughput(desiredThroughput);
throughputResponse = await container.ReplaceThroughputAsync(newThroughputProperties);
WriteLineInColor($"\nChanged {container.Id}'s max throughput to {desiredThroughput}", ConsoleColor.Cyan);
}
}
else
{
// Container configured with manual throughput
if (throughputProperties.Throughput.HasValue && throughputProperties.Throughput.Value == desiredThroughput)
{
WriteLineInColor($"\nThe {container.Id} container is already configured with throughput: {desiredThroughput}. Skipping performance change request...", ConsoleColor.Yellow);
}
else
{
WriteLineInColor($"\nThe {container.Id} container is configured with throughput: {throughputProperties.AutoscaleMaxThroughput}\nChanging throughput to {desiredThroughput}", ConsoleColor.White);
ThroughputProperties newThroughputProperties = ThroughputProperties.CreateManualThroughput(desiredThroughput);
throughputResponse = await container.ReplaceThroughputAsync(newThroughputProperties);
WriteLineInColor($"\nChanged {container.Id}'s throughput to {desiredThroughput}", ConsoleColor.Cyan);
}
}
}
[TestCategory("Quarantine")] // Not currently working with emulator
public async Task ReadFixedWithAutoscaleTests()
{
DatabaseInternal database = (DatabaseInlineCore)await this.cosmosClient.CreateDatabaseAsync(
Guid.NewGuid().ToString());
ContainerInternal autoscaleContainer = (ContainerInlineCore)await database.CreateContainerAsync(
new ContainerProperties(Guid.NewGuid().ToString(), "/pk"),
ThroughputProperties.CreateAutoscaleThroughput(5000));
Assert.IsNotNull(autoscaleContainer);
// Reading a autoscale container with fixed results
int?throughput = await autoscaleContainer.ReadThroughputAsync();
Assert.IsNotNull(throughput);
await database.DeleteAsync();
}
public override async Task <ContainerResponse> CreateContainerAsync(
ContainerProperties containerProperties,
ThroughputProperties throughputProperties,
RequestOptions requestOptions = null,
CancellationToken cancellationToken = default)
{
ContainerResponse containerResponse = await this.database.CreateContainerAsync(
containerProperties,
throughputProperties,
requestOptions,
cancellationToken);
EncryptionContainerResponse encryptionContainerResponse = new EncryptionContainerResponse(
containerResponse,
new EncryptionContainer(containerResponse, this.EncryptionCosmosClient));
return(encryptionContainerResponse);
}
// </CreateContainer>
// <CreateAndUpdateAutoscaleContainer>
private static async Task CreateAndUpdateAutoscaleContainer()
{
// Set autoscale throughput to the maximum value of 10000 RU/s
ContainerProperties containerProperties = new ContainerProperties(autoscaleContainerId, partitionKeyPath);
Container autoscaleContainer = await database.CreateContainerIfNotExistsAsync(
containerProperties : containerProperties,
throughputProperties : ThroughputProperties.CreateAutoscaleThroughput(autoscaleMaxThroughput: 10000));
Console.WriteLine($"{Environment.NewLine}1.2. Created autoscale container :{autoscaleContainer.Id}");
//*********************************************************************************************
// Get configured performance of a CosmosContainer
//**********************************************************************************************
ThroughputResponse throughputResponse = await autoscaleContainer.ReadThroughputAsync(requestOptions : null);
Console.WriteLine($"{Environment.NewLine}1.2.1. Found autoscale throughput {Environment.NewLine}The current throughput: {throughputResponse.Resource.Throughput} Max throughput: {throughputResponse.Resource.AutoscaleMaxThroughput} " +
$"using container's id: {autoscaleContainer.Id}");
//*********************************************************************************************
// Get the current throughput configured for a Container
//**********************************************************************************************
int?currentThroughput = await autoscaleContainer.ReadThroughputAsync();
Console.WriteLine($"{Environment.NewLine}1.2.2. Found autoscale throughput {Environment.NewLine}The current throughput: {currentThroughput} using container's id: {autoscaleContainer.Id}");
//******************************************************************************************************************
// Change performance (reserved throughput) of CosmosContainer
// Let's change the performance of the autoscale container to a maximum throughput of 15000 RU/s
//******************************************************************************************************************
ThroughputResponse throughputUpdateResponse = await autoscaleContainer.ReplaceThroughputAsync(ThroughputProperties.CreateAutoscaleThroughput(15000));
Console.WriteLine($"{Environment.NewLine}1.2.3. Replaced autoscale throughput. {Environment.NewLine}The current throughput: {throughputUpdateResponse.Resource.Throughput} Max throughput: {throughputUpdateResponse.Resource.AutoscaleMaxThroughput} " +
$"using container's id: {autoscaleContainer.Id}");
// Get the offer again after replace
throughputResponse = await autoscaleContainer.ReadThroughputAsync(requestOptions : null);
Console.WriteLine($"{Environment.NewLine}1.2.4. Found autoscale throughput {Environment.NewLine}The current throughput: {throughputResponse.Resource.Throughput} Max throughput: {throughputResponse.Resource.AutoscaleMaxThroughput} " +
$"using container's id: {autoscaleContainer.Id}{Environment.NewLine}");
// Delete the container
await autoscaleContainer.DeleteContainerAsync();
}
public async Task CreateEventStoreAsync(
ThroughputProperties throughputProperties,
CancellationToken cancellationToken)
{
await clientFactory
.GetClient()
.CreateDatabaseIfNotExistsAsync(
options.EventStoreDatabaseId,
throughputProperties,
cancellationToken: cancellationToken)
.ConfigureAwait(false);
await CreateEventStoreContainerAsync(cancellationToken)
.ConfigureAwait(continueOnCapturedContext: false);
await CreateSubscriptionContainerAsync(cancellationToken)
.ConfigureAwait(continueOnCapturedContext: false);
await CreateIndexContainerAsync(cancellationToken)
.ConfigureAwait(continueOnCapturedContext: false);
}
[TestCategory("Quarantine")] // Not currently working with emulator
public async Task ContainerAutoscaleIfExistsTest()
{
DatabaseInternal database = (DatabaseInlineCore)await this.cosmosClient.CreateDatabaseAsync(
nameof(CreateDropAutoscaleDatabase) + Guid.NewGuid().ToString());
Container container = await database.CreateContainerAsync(
containerProperties : new ContainerProperties("Test", "/id"),
throughputProperties : ThroughputProperties.CreateAutoscaleThroughput(5000));
ContainerInternal containerCore = (ContainerInlineCore)container;
ThroughputResponse throughputResponse = await database.ReadThroughputIfExistsAsync(requestOptions : null);
Assert.IsNotNull(throughputResponse);
Assert.AreEqual(HttpStatusCode.NotFound, throughputResponse.StatusCode);
Assert.IsNull(throughputResponse.Resource);
throughputResponse = await database.ReplaceThroughputPropertiesIfExistsAsync(
ThroughputProperties.CreateAutoscaleThroughput(6000));
Assert.IsNotNull(throughputResponse);
Assert.AreEqual(HttpStatusCode.NotFound, throughputResponse.StatusCode);
Assert.IsNull(throughputResponse.Resource);
throughputResponse = await containerCore.ReadThroughputIfExistsAsync(
requestOptions : null,
default(CancellationToken));
Assert.IsNotNull(throughputResponse);
Assert.IsTrue(throughputResponse.Resource.Throughput > 400);
Assert.AreEqual(5000, throughputResponse.Resource.AutoscaleMaxThroughput);
throughputResponse = await containerCore.ReplaceThroughputIfExistsAsync(
ThroughputProperties.CreateAutoscaleThroughput(6000),
requestOptions : null,
default(CancellationToken));
Assert.IsNotNull(throughputResponse);
Assert.IsTrue(throughputResponse.Resource.Throughput > 400);
Assert.AreEqual(6000, throughputResponse.Resource.AutoscaleMaxThroughput);
await database.DeleteAsync();
}
public async Task CreateDropAutoscaleDatabase()
{
DatabaseInternal database = (DatabaseInlineCore)await this.cosmosClient.CreateDatabaseAsync(
nameof(CreateDropAutoscaleDatabase) + Guid.NewGuid().ToString(),
ThroughputProperties.CreateAutoscaleThroughput(5000));
ThroughputResponse autoscale = await database.ReadThroughputAsync(requestOptions : null);
Assert.IsNotNull(autoscale);
Assert.AreEqual(5000, autoscale.Resource.AutoscaleMaxThroughput);
ThroughputResponse autoscaleReplaced = await database.ReplaceThroughputAsync(
ThroughputProperties.CreateAutoscaleThroughput(10000));
Assert.IsNotNull(autoscaleReplaced);
Assert.AreEqual(10000, autoscaleReplaced.Resource.AutoscaleMaxThroughput);
await database.DeleteAsync();
}
public async Task <DatabaseResponse> CreateDatabase(string name, int sharedRu)
{
if (sharedRu > 0)
{
if (sharedRu < 4000) // min sharedRu value is 4000
{
ThroughputProperties props = ThroughputProperties.CreateAutoscaleThroughput(4000);
return(await client.CreateDatabaseIfNotExistsAsync(name, throughputProperties : props));
}
else
{
ThroughputProperties props = ThroughputProperties.CreateAutoscaleThroughput(sharedRu);
return(await client.CreateDatabaseIfNotExistsAsync(name, throughputProperties : props));
}
}
else
{
return(await client.CreateDatabaseIfNotExistsAsync(name));
}
}
private static async Task InitializeCosmosDb()
{
Database database = await _cosmosDbClient.CreateDatabaseIfNotExistsAsync(DatabaseName);
// Create a partition based on the ipCountryCode value from the Transactions data set.
// This partition was selected because the data will most likely include this value, and
// it allows us to partition by location from which the transaction originated. This field
// also contains a wide range of values, which is preferable for partitions.
var containerProperties = new ContainerProperties(CollectionName, PartitionKey)
{
// The default indexing policy for newly created containers is set to automatic and enforces
// range indexes for any string or number.
IndexingPolicy = { Automatic = true }
};
// Create the container with a maximum autoscale throughput of 4000 RU/s. By default, the
// minimum RU/s will be 400.
var container = await database.CreateContainerIfNotExistsAsync(
containerProperties : containerProperties,
throughputProperties : ThroughputProperties.CreateAutoscaleThroughput(autoscaleMaxThroughput: 4000));
}
public async Task SC00_CreateDB()
{
CosmosClient client = new CosmosClient(EndpointUrl, AuthorizationKey);
await client.CreateDatabaseIfNotExistsAsync(DatabaseId, ThroughputProperties.CreateManualThroughput(400));
Database database = client.GetDatabase(DatabaseId);
await database.DefineContainer(_eventsContainerId, "/stream/id").CreateIfNotExistsAsync();
await database.DefineContainer(_newEventsContainerId, "/stream/id").CreateIfNotExistsAsync();
await database.DefineContainer(_leaseContainerId, "/id").CreateIfNotExistsAsync();
await database.DefineContainer(_viewsContainerId, "/id").CreateIfNotExistsAsync();
await database.DefineContainer(_snapshotContainerId, "/id").CreateIfNotExistsAsync();
await AddStoredProc(database, _eventsContainerId);
await AddStoredProc(database, _newEventsContainerId);
}
private ThroughputProperties GetThroughputProperties()
{
string throughput;
ThroughputProperties properties = null;
if (!string.IsNullOrEmpty(this.Configuration.GetValueOrDefault("AutoscaleThroughput")))
{
throughput = this.Configuration.GetValueOrDefault("AutoscaleThroughput");
bool valid = int.TryParse(throughput, out int autoscaleThroughput);
if (valid)
{
properties = ThroughputProperties.CreateAutoscaleThroughput(autoscaleThroughput);
return(properties);
}
}
if (!string.IsNullOrEmpty(this.Configuration.GetValueOrDefault("ManualThroughput")))
{
throughput = this.Configuration.GetValueOrDefault("ManualThroughput");
bool valid = int.TryParse(throughput, out int manualThroughput);
if (valid)
{
properties = ThroughputProperties.CreateManualThroughput(manualThroughput);
return(properties);
}
}
properties = ThroughputProperties.CreateAutoscaleThroughput(4000);
return(properties);
}
public async Task DatabaseAutoscaleIfExistsTest()
{
DatabaseInternal database = (DatabaseInlineCore)await this.cosmosClient.CreateDatabaseAsync(
nameof(CreateDropAutoscaleDatabase) + Guid.NewGuid().ToString(),
ThroughputProperties.CreateAutoscaleThroughput(5000));
Container container = await database.CreateContainerAsync("Test", "/id");
ContainerInternal containerCore = (ContainerInlineCore)container;
ThroughputResponse throughputResponse = await database.ReadThroughputIfExistsAsync(requestOptions : null);
Assert.IsNotNull(throughputResponse);
Assert.AreEqual(5000, throughputResponse.Resource.AutoscaleMaxThroughput);
throughputResponse = await database.ReplaceThroughputAsync(
ThroughputProperties.CreateAutoscaleThroughput(6000));
Assert.IsNotNull(throughputResponse);
Assert.AreEqual(6000, throughputResponse.Resource.AutoscaleMaxThroughput);
throughputResponse = await containerCore.ReadThroughputIfExistsAsync(
requestOptions : null,
public async Task DatabaseAutoscaleIfExistsTest()
{
DatabaseCore database = (DatabaseInlineCore)await this.cosmosClient.CreateDatabaseAsync(
nameof(CreateDropAutoscaleDatabase) + Guid.NewGuid().ToString(),
ThroughputProperties.CreateAutoscaleProvionedThroughput(5000));
Container container = await database.CreateContainerAsync("Test", "/id");
ContainerCore containerCore = (ContainerInlineCore)container;
ThroughputResponse throughputResponse = await database.ReadThroughputIfExistsAsync(requestOptions : null);
Assert.IsNotNull(throughputResponse);
Assert.AreEqual(5000, throughputResponse.Resource.MaxAutoscaleThroughput);
throughputResponse = await database.ReplaceThroughputPropertiesAsync(
ThroughputProperties.CreateAutoscaleProvionedThroughput(6000));
Assert.IsNotNull(throughputResponse);
Assert.AreEqual(6000, throughputResponse.Resource.MaxAutoscaleThroughput);
throughputResponse = await containerCore.ReadThroughputIfExistsAsync(requestOptions : null);
Assert.IsNotNull(throughputResponse);
Assert.AreEqual(HttpStatusCode.NotFound, throughputResponse.StatusCode);
Assert.IsNull(throughputResponse.Resource);
throughputResponse = await containerCore.ReplaceThroughputPropertiesIfExistsAsync(
ThroughputProperties.CreateAutoscaleProvionedThroughput(6000));
Assert.IsNotNull(throughputResponse);
Assert.AreEqual(HttpStatusCode.NotFound, throughputResponse.StatusCode);
Assert.IsNull(throughputResponse.Resource);
await database.DeleteAsync();
}
public async Task CreateContainerIfNotExistTest()
{
string dbName = nameof(CreateContainerIfNotExistTest) + Guid.NewGuid();
DatabaseCore database = (DatabaseInlineCore)await this.cosmosClient.CreateDatabaseAsync(dbName);
ContainerProperties containerProperties = new ContainerProperties("Test", "/id");
ContainerResponse containerResponse = await database.CreateContainerIfNotExistsAsync(
containerProperties,
ThroughputProperties.CreateAutoscaleThroughput(5000));
Assert.AreEqual(HttpStatusCode.Created, containerResponse.StatusCode);
ThroughputResponse autoscale = await containerResponse.Container.ReadThroughputAsync(requestOptions : null);
Assert.IsNotNull(autoscale);
Assert.IsNotNull(autoscale.Resource.Throughput);
Assert.AreEqual(5000, autoscale.Resource.AutoscaleMaxThroughput);
containerResponse = await database.CreateContainerIfNotExistsAsync(
containerProperties,
ThroughputProperties.CreateAutoscaleThroughput(5000));
Assert.AreEqual(HttpStatusCode.OK, containerResponse.StatusCode);
}
[TestCategory("Quarantine")] // Not currently working with emulator
public async Task CreateDropAutoscaleContainer()
{
DatabaseCore database = (DatabaseInlineCore)await this.cosmosClient.CreateDatabaseAsync(
Guid.NewGuid().ToString());
ContainerCore container = (ContainerInlineCore)await database.CreateContainerAsync(
new ContainerProperties(Guid.NewGuid().ToString(), "/pk"),
ThroughputProperties.CreateAutoscaleProvionedThroughput(5000));
Assert.IsNotNull(container);
ThroughputResponse throughputResponse = await container.ReadThroughputAsync(requestOptions : null);
Assert.IsNotNull(throughputResponse);
Assert.AreEqual(5000, throughputResponse.Resource.MaxAutoscaleThroughput);
throughputResponse = await container.ReplaceThroughputAsync(
ThroughputProperties.CreateAutoscaleProvionedThroughput(6000));
Assert.IsNotNull(throughputResponse);
Assert.AreEqual(6000, throughputResponse.Resource.MaxAutoscaleThroughput);
await database.DeleteAsync();
}
/// <summary>
/// Creates a new database
/// </summary>
private async Task CreateDatabaseIfNotExistsAsync()
{
var throughputProperties = ThroughputProperties.CreateAutoscaleThroughput(MaxAutoScaleThroughput);
await cosmosClient.CreateDatabaseIfNotExistsAsync(container.Database.Id, throughputProperties);
}
private static async Task Main(string[] args)
{
// go over all galleries and images with comments and update the gallery comment count
// used as a one-off to set the initial values for this new property, though could potentially be
// modified to re-calculate all counts if need be.
// oct '21: modified to fill in null values
_configuration = new ConfigurationBuilder()
.AddJsonFile(args[0], optional: false)
.Build();
// initialise the Application Server
await Server.Instance.SetConfigurationAsync(_configuration);
// authenticate with the CosmosDB service and create a client we can re-use
_cosmosClient = new CosmosClient(_configuration["CosmosDB:Uri"], _configuration["CosmosDB:PrimaryKey"]);
// create the CosmosDB database if it doesn't already exist
var response = await _cosmosClient.CreateDatabaseIfNotExistsAsync(_configuration["CosmosDB:DatabaseName"], ThroughputProperties.CreateManualThroughput(400));
// keep a reference to the database so other parts of the app can easily interact with it
_database = response.Database;
_imagesContainer = _database.GetContainer(Constants.ImagesContainerName);
_galleriesContainer = _database.GetContainer(Constants.GalleriesContainerName);
// query the database for galleries without CommentCount, set their values to zero
const string galleryQuery = "SELECT g.id, g.CategoryId FROM Galleries g WHERE NOT IS_DEFINED(g.CommentComment)";
var galleryQueryDefinition = new QueryDefinition(galleryQuery);
Console.WriteLine("Getting gallery stubs...");
var galleryQueryResult = _galleriesContainer.GetItemQueryIterator <GalleryCommentCountStub>(galleryQueryDefinition);
Console.WriteLine("Got gallery stubs. Enumerating...");
while (galleryQueryResult.HasMoreResults)
{
var galleryCommentCountStubs = await galleryQueryResult.ReadNextAsync();
foreach (var galleryCommentCountStub in galleryCommentCountStubs)
{
var gallery = await Server.Instance.Galleries.GetGalleryAsync(galleryCommentCountStub.CategoryId, galleryCommentCountStub.Id);
gallery.CommentCount = 0;
await Server.Instance.Galleries.UpdateGalleryAsync(gallery);
Console.WriteLine("Gallery comments set to zero for gallery id: " + galleryCommentCountStub.Id);
}
}
// query the database for images without CommentCount values, then set them to zero
const string imageQuery = "SELECT i.id, i.GalleryId FROM Images i WHERE NOT IS_DEFINED(i.CommentCount)";
var imageQueryDefinition = new QueryDefinition(imageQuery);
Console.WriteLine("Getting image stubs...");
var imageQueryResult = _imagesContainer.GetItemQueryIterator <ImageCommentCountStub>(imageQueryDefinition);
Console.WriteLine("Got image stubs. Enumerating...");
while (imageQueryResult.HasMoreResults)
{
var imageCommentCountStubs = await imageQueryResult.ReadNextAsync();
foreach (var imageCommentCountStub in imageCommentCountStubs)
{
var image = await Server.Instance.Images.GetImageAsync(imageCommentCountStub.GalleryId, imageCommentCountStub.Id);
image.CommentCount = 0;
await Server.Instance.Images.UpdateImageAsync(image);
Console.WriteLine($"Set image CommentCount to zero for image id: {image.Id}");
}
}
}
public void ValidateCustomSerializerNotUsedForInternalTypes()
{
CosmosSerializerHelper serializerHelper = new CosmosSerializerHelper(
null,
(item) => throw new ArgumentException("Should be using internal serializer"),
(item) => throw new ArgumentException("Should be using internal serializer"));
CosmosSerializerCore serializerCore = new CosmosSerializerCore(serializerHelper);
string id = "testId";
this.TestProperty <AccountProperties>(
serializerCore,
id,
$@"{{""id"":""{id}"",""writableLocations"":[],""readableLocations"":[],""userConsistencyPolicy"":null,""addresses"":null,""userReplicationPolicy"":null,""systemReplicationPolicy"":null,""readPolicy"":null,""queryEngineConfiguration"":null,""enableMultipleWriteLocations"":false}}");
this.TestProperty <DatabaseProperties>(
serializerCore,
id,
$@"{{""id"":""{id}""}}");
this.TestProperty <ContainerProperties>(
serializerCore,
id,
$@"{{""id"":""{id}"",""partitionKey"":{{""paths"":[],""kind"":""Hash""}}}}");
this.TestProperty <StoredProcedureProperties>(
serializerCore,
id,
$@"{{""body"":""bodyCantBeNull"",""id"":""{id}""}}");
this.TestProperty <TriggerProperties>(
serializerCore,
id,
$@"{{""body"":null,""triggerType"":""Pre"",""triggerOperation"":""All"",""id"":""{id}""}}");
this.TestProperty <UserDefinedFunctionProperties>(
serializerCore,
id,
$@"{{""body"":null,""id"":""{id}""}}");
this.TestProperty <UserProperties>(
serializerCore,
id,
$@"{{""id"":""{id}"",""_permissions"":null}}");
this.TestProperty <PermissionProperties>(
serializerCore,
id,
$@"{{""id"":""{id}"",""resource"":null,""permissionMode"":0}}");
this.TestProperty <ConflictProperties>(
serializerCore,
id,
$@"{{""id"":""{id}"",""operationType"":""Invalid"",""resourceType"":null,""resourceId"":null,""content"":null,""conflict_lsn"":0}}");
// Throughput doesn't have an id.
string defaultThroughputJson = @"{}";
ThroughputProperties property = JsonConvert.DeserializeObject <ThroughputProperties>(defaultThroughputJson);
Assert.IsNull(property.Throughput);
string propertyJson = JsonConvert.SerializeObject(property);
Assert.AreEqual(defaultThroughputJson, propertyJson);
}
public async Task AutoscaleThroughputSerializationTest()
{
ThroughputProperties autoscaleThroughputProperties = ThroughputProperties.CreateAutoscaleThroughput(1000);
Assert.AreEqual(1000, autoscaleThroughputProperties.AutoscaleMaxThroughput);
Assert.IsNull(autoscaleThroughputProperties.Throughput);
using (Stream stream = MockCosmosUtil.Serializer.ToStream <ThroughputProperties>(autoscaleThroughputProperties))
{
using (StreamReader reader = new StreamReader(stream))
{
string output = await reader.ReadToEndAsync();
Assert.IsNotNull(output);
Assert.AreEqual("{\"content\":{\"offerAutopilotSettings\":{\"maxThroughput\":1000}},\"offerVersion\":\"V2\"}", output);
}
}
OfferAutoscaleProperties autoscaleProperties = new OfferAutoscaleProperties(
startingMaxThroughput: 1000,
autoUpgradeMaxThroughputIncrementPercentage: null);
using (Stream stream = MockCosmosUtil.Serializer.ToStream <OfferAutoscaleProperties>(autoscaleProperties))
{
using (StreamReader reader = new StreamReader(stream))
{
string output = await reader.ReadToEndAsync();
Assert.IsNotNull(output);
Assert.AreEqual("{\"maxThroughput\":1000}", output);
}
}
using (Stream stream = MockCosmosUtil.Serializer.ToStream <OfferAutoscaleProperties>(autoscaleProperties))
{
OfferAutoscaleProperties fromStream = MockCosmosUtil.Serializer.FromStream <OfferAutoscaleProperties>(stream);
Assert.IsNotNull(fromStream);
Assert.AreEqual(1000, fromStream.MaxThroughput);
}
OfferContentProperties content = OfferContentProperties.CreateAutoscaleOfferConent(
startingMaxThroughput: 1000,
autoUpgradeMaxThroughputIncrementPercentage: null);
using (Stream stream = MockCosmosUtil.Serializer.ToStream <OfferContentProperties>(content))
{
using (StreamReader reader = new StreamReader(stream))
{
string output = await reader.ReadToEndAsync();
Assert.IsNotNull(output);
Assert.AreEqual("{\"offerAutopilotSettings\":{\"maxThroughput\":1000}}", output);
}
}
using (Stream stream = MockCosmosUtil.Serializer.ToStream <OfferContentProperties>(content))
{
OfferContentProperties fromStream = MockCosmosUtil.Serializer.FromStream <OfferContentProperties>(stream);
Assert.IsNotNull(fromStream.OfferAutoscaleSettings);
Assert.AreEqual(1000, fromStream.OfferAutoscaleSettings.MaxThroughput);
Assert.IsNull(fromStream.OfferThroughput);
}
using (Stream stream = MockCosmosUtil.Serializer.ToStream <ThroughputProperties>(autoscaleThroughputProperties))
{
ThroughputProperties fromStream = MockCosmosUtil.Serializer.FromStream <ThroughputProperties>(stream);
Assert.AreEqual(1000, fromStream.AutoscaleMaxThroughput);
Assert.IsNull(fromStream.Throughput);;
}
}
/// <inheritdoc />
public async Task InitializeDataStore(CosmosClient client, CosmosDataStoreConfiguration cosmosDataStoreConfiguration, IEnumerable <ICollectionInitializer> collectionInitializers)
{
EnsureArg.IsNotNull(client, nameof(client));
EnsureArg.IsNotNull(cosmosDataStoreConfiguration, nameof(cosmosDataStoreConfiguration));
EnsureArg.IsNotNull(collectionInitializers, nameof(collectionInitializers));
try
{
_logger.LogInformation("Initializing Cosmos DB Database {DatabaseId} and collections", cosmosDataStoreConfiguration.DatabaseId);
if (cosmosDataStoreConfiguration.AllowDatabaseCreation)
{
_logger.LogInformation("CreateDatabaseIfNotExists {DatabaseId}", cosmosDataStoreConfiguration.DatabaseId);
await _retryExceptionPolicyFactory.GetRetryPolicy().ExecuteAsync(
async() =>
await client.CreateDatabaseIfNotExistsAsync(
cosmosDataStoreConfiguration.DatabaseId,
cosmosDataStoreConfiguration.InitialDatabaseThroughput.HasValue ? ThroughputProperties.CreateManualThroughput(cosmosDataStoreConfiguration.InitialDatabaseThroughput.Value) : null));
}
foreach (var collectionInitializer in collectionInitializers)
{
await collectionInitializer.InitializeCollection(client);
}
_logger.LogInformation("Cosmos DB Database {DatabaseId} and collections successfully initialized", cosmosDataStoreConfiguration.DatabaseId);
}
catch (Exception ex)
{
LogLevel logLevel = ex is RequestRateExceededException ? LogLevel.Warning : LogLevel.Critical;
_logger.Log(logLevel, ex, "Cosmos DB Database {DatabaseId} and collections initialization failed", cosmosDataStoreConfiguration.DatabaseId);
throw;
}
}
public async Task ReadReplaceThroughputResponseTests()
{
int toStreamCount = 0;
int fromStreamCount = 0;
CosmosSerializerHelper mockJsonSerializer = new CosmosSerializerHelper(
null,
(x) => fromStreamCount++,
(x) => toStreamCount++);
//Create a new cosmos client with the mocked cosmos json serializer
CosmosClient client = TestCommon.CreateCosmosClient(
(cosmosClientBuilder) => cosmosClientBuilder.WithCustomSerializer(mockJsonSerializer));
string databaseId = Guid.NewGuid().ToString();
int throughput = 10000;
DatabaseResponse createResponse = await client.CreateDatabaseAsync(databaseId, throughput, null);
Assert.AreEqual(HttpStatusCode.Created, createResponse.StatusCode);
Cosmos.Database cosmosDatabase = createResponse;
ThroughputResponse readThroughputResponse = await cosmosDatabase.ReadThroughputAsync(new RequestOptions());
Assert.IsNotNull(readThroughputResponse);
Assert.IsNotNull(readThroughputResponse.Resource);
Assert.IsNotNull(readThroughputResponse.MinThroughput);
Assert.IsNotNull(readThroughputResponse.Resource.Throughput);
Assert.AreEqual(throughput, readThroughputResponse.Resource.Throughput.Value);
// Implicit
ThroughputProperties throughputProperties = await cosmosDatabase.ReadThroughputAsync(new RequestOptions());
Assert.IsNotNull(throughputProperties);
Assert.AreEqual(throughput, throughputProperties.Throughput);
// Simple API
int?readThroughput = await cosmosDatabase.ReadThroughputAsync();
Assert.AreEqual(throughput, readThroughput);
// Database must have a container before it can be scaled
string containerId = Guid.NewGuid().ToString();
string partitionPath = "/users";
ContainerResponse containerResponse = await cosmosDatabase.CreateContainerAsync(containerId, partitionPath, throughput : null);
Assert.AreEqual(HttpStatusCode.Created, containerResponse.StatusCode);
ThroughputResponse replaceThroughputResponse = await cosmosDatabase.ReplaceThroughputAsync(readThroughputResponse.Resource.Throughput.Value + 1000);
Assert.IsNotNull(replaceThroughputResponse);
Assert.IsNotNull(replaceThroughputResponse.Resource);
Assert.AreEqual(readThroughputResponse.Resource.Throughput.Value + 1000, replaceThroughputResponse.Resource.Throughput.Value);
await cosmosDatabase.DeleteAsync();
Database databaseNoThroughput = await client.CreateDatabaseAsync(Guid.NewGuid().ToString(), throughput : null);
try
{
ThroughputResponse throughputResponse = await databaseNoThroughput.ReadThroughputAsync(new RequestOptions());
Assert.Fail("Should through not found exception as throughput is not configured");
}
catch (CosmosException exception)
{
Assert.AreEqual(HttpStatusCode.NotFound, exception.StatusCode);
}
try
{
ThroughputResponse throughputResponse = await databaseNoThroughput.ReplaceThroughputAsync(2000, new RequestOptions());
Assert.Fail("Should through not found exception as throughput is not configured");
}
catch (CosmosException exception)
{
Assert.AreEqual(HttpStatusCode.NotFound, exception.StatusCode);
}
int?dbThroughput = await databaseNoThroughput.ReadThroughputAsync();
Assert.IsNull(dbThroughput);
Assert.AreEqual(0, toStreamCount, "Custom serializer to stream should not be used for offer operations");
Assert.AreEqual(0, fromStreamCount, "Custom serializer from stream should not be used for offer operations");
await databaseNoThroughput.DeleteAsync();
}
private static async Task CreateImpl()
{
var cosmosClient = new CosmosClient("https://localhost:8081", "C2y6yDjf5/R+ob0N8A7Cgv30VRDJIWEHLM+4QDU5DE2nQ9nDuVTqobD4b8mGGyPMbIZnqyMsEcaGQy67XIw/Jw==");
var database = await cosmosClient.CreateDatabaseIfNotExistsAsync("db", ThroughputProperties.CreateAutoscaleThroughput(40000));
await database.Database.CreateContainerIfNotExistsAsync("graph", "/PartitionKey");
}
