public async Task Setup_CosmosResources()
{
// Create resources for Scenario 0 - setup
Console.BackgroundColor = ConsoleColor.Blue;
Console.ForegroundColor = ConsoleColor.Yellow;
Console.WriteLine("Running setup...\n");
Console.ResetColor();
// Create database for partitioning demo. We set the RU/s high to simulate having a large workload with many partitions, to demo the impact of cross-partition queries on larger datasets. If you do not wish to demo Scenario 2, replace the throughput with the minimum of 400 RU/s.
var partitionDemoDatabaseName = "PartitioningDemo";
Microsoft.Azure.Cosmos.Database partitioningDemoDB = await cosmosClient.CreateDatabaseIfNotExistsAsync(id : partitionDemoDatabaseName, throughput : 40000);
Console.WriteLine("\tCreated database {0}", partitioningDemoDB.Id);
// Create 2 containers with different partition keys for Scenarios
UserReviewsV1Container = await partitioningDemoDB.CreateContainerIfNotExistsAsync(id : "UserReviews_v1", partitionKeyPath : "/id");
Console.WriteLine("\tCreated container {0} with partition key 'id'", UserReviewsV1Container.Id);
UserReviewsV2Container = await partitioningDemoDB.CreateContainerIfNotExistsAsync(id : "UserReviews_v2", partitionKeyPath : "/username");
Console.WriteLine("\tCreated container {0} with partition key 'username'", UserReviewsV2Container.Id);
// Create database Indexing policy demo
var indexingDemoDatabaseName = "IndexingDemo";
Microsoft.Azure.Cosmos.Database indexingDemoDB = await cosmosClient.CreateDatabaseIfNotExistsAsync(id : indexingDemoDatabaseName, throughput : 400);
// Create containers for Indexing policy demo
var defaultIndexingPolicyContatinerName = "DefaultIndexingPolicy";
var tunedIndexingPolicyContainerName = "TunedIndexingPolicy";
ContainerResponse defaultIndexingPolicyContainer = await indexingDemoDB.DefineContainer(name : defaultIndexingPolicyContatinerName, partitionKeyPath : "/partitionKey")
.CreateIfNotExistsAsync();
Console.WriteLine("\tCreated container {0} with default indexing policy", defaultIndexingPolicyContainer.Resource.Id);
ContainerResponse tunedIndexingPolicyContainer = await indexingDemoDB.DefineContainer(name : tunedIndexingPolicyContainerName, partitionKeyPath : "/partitionKey")
// Define indexing policy with included and excluded paths
.WithIndexingPolicy()
.WithIncludedPaths()
.Path("/username/*")
.Path("/productName/*")
.Path("/verifiedPurchase/*")
.Attach()
.WithExcludedPaths()
.Path("/*")
.Attach()
.Attach()
.CreateIfNotExistsAsync();
Console.WriteLine("\tCreated container {0} with tuned indexing policy", tunedIndexingPolicyContainer.Resource.Id);
// Import data into the containers
Console.BackgroundColor = ConsoleColor.Blue;
Console.ForegroundColor = ConsoleColor.Yellow;
Console.WriteLine("\nStarting data import: this will take a few minutes...");
Console.ResetColor();
using (WebClient wc = new WebClient())
{
var json = wc.DownloadString("https://cosmosnotebooksdata.blob.core.windows.net/notebookdata/ProductReviewsData.json");
List <dynamic> userProductReviews = JsonConvert.DeserializeObject <List <dynamic> >(json);
Console.WriteLine("\tFinished downloading data from 'https://cosmosnotebooksdata.blob.core.windows.net/notebookdata/ProductReviewsData.json'");
Console.WriteLine("\tWriting data into Cosmos containers...");
Stopwatch stopWatch = new Stopwatch();
stopWatch.Start();
// Enable bulk execution mode for faster import
cosmosClient.ClientOptions.AllowBulkExecution = true;
List <Task <int> > tasks = new List <Task <int> >();
for (var i = 0; i < userProductReviews.Count; i++)
{
var documentType = userProductReviews[i].documentType;
if (documentType == "user")
{
User user = JsonConvert.DeserializeObject <User>(userProductReviews[i].ToString());
tasks.Add(InsertUserItemsAsync(user));
}
else if (documentType == "review")
{
ProductReview productReview = JsonConvert.DeserializeObject <ProductReview>(userProductReviews[i].ToString());
tasks.Add(InsertProductReviewItemsAsync(productReview));
}
}
await Task.WhenAll(tasks);
stopWatch.Stop();
TimeSpan ts = stopWatch.Elapsed;
//Print results
string elapsedTime = String.Format("{0:00}:{1:00}:{2:00}.{3:00}",
ts.Hours, ts.Minutes, ts.Seconds,
ts.Milliseconds / 10);
Console.WriteLine("\tImport of {0} items into {1} and {2} containers completed. Total time: {3}", tasks.Sum(task => task.Result) / 2, UserReviewsV1Container.Id, UserReviewsV2Container.Id, elapsedTime);
}
// Scale down the database to the minimum RU/s.
var currentThroughput = await partitioningDemoDB.ReadThroughputAsync(requestOptions : new RequestOptions());
int minThroughput = (int)currentThroughput.MinThroughput;
var throughputResponse = await partitioningDemoDB.ReplaceThroughputAsync(minThroughput);
Console.WriteLine("\tScaled PartitioningDemo database down to minimum {0} RU/s", throughputResponse.Resource.Throughput);
// Reset bulk execution mode to false for rest of demo
cosmosClient.ClientOptions.AllowBulkExecution = false;
}
/// <summary>
/// Run samples for Order By queries.
/// </summary>
/// <returns>a Task object.</returns>
private async Task <RunSummary> ExecuteAsync(BenchmarkConfig config)
{
using (CosmosClient cosmosClient = config.CreateCosmosClient(config.Key))
{
Microsoft.Azure.Cosmos.Database database = cosmosClient.GetDatabase(config.Database);
if (config.CleanupOnStart)
{
await database.DeleteStreamAsync();
}
ContainerResponse containerResponse = await Program.CreatePartitionedContainerAsync(config, cosmosClient);
Container container = containerResponse;
int?currentContainerThroughput = await container.ReadThroughputAsync();
if (!currentContainerThroughput.HasValue)
{
// Container throughput is not configured. It is shared database throughput
ThroughputResponse throughputResponse = await database.ReadThroughputAsync(requestOptions : null);
throw new InvalidOperationException($"Using database {config.Database} with {throughputResponse.Resource.Throughput} RU/s. " +
$"Container {config.Container} must have a configured throughput.");
}
Console.WriteLine($"Using container {config.Container} with {currentContainerThroughput} RU/s");
int taskCount = config.GetTaskCount(currentContainerThroughput.Value);
Console.WriteLine("Starting Inserts with {0} tasks", taskCount);
Console.WriteLine();
string partitionKeyPath = containerResponse.Resource.PartitionKeyPath;
int opsPerTask = config.ItemCount / taskCount;
// TBD: 2 clients SxS some overhead
RunSummary runSummary;
using (DocumentClient documentClient = config.CreateDocumentClient(config.Key))
{
Func <IBenchmarkOperation> benchmarkOperationFactory = this.GetBenchmarkFactory(
config,
partitionKeyPath,
cosmosClient,
documentClient);
if (config.DisableCoreSdkLogging)
{
// Do it after client initialization (HACK)
Program.ClearCoreSdkListeners();
}
IExecutionStrategy execution = IExecutionStrategy.StartNew(config, benchmarkOperationFactory);
runSummary = await execution.ExecuteAsync(taskCount, opsPerTask, config.TraceFailures, 0.01);
}
if (config.CleanupOnFinish)
{
Console.WriteLine($"Deleting Database {config.Database}");
await database.DeleteStreamAsync();
}
runSummary.WorkloadType = config.WorkloadType;
runSummary.id = $"{DateTime.UtcNow:yyyy-MM-dd:HH-mm}-{config.CommitId}";
runSummary.Commit = config.CommitId;
runSummary.CommitDate = config.CommitDate;
runSummary.CommitTime = config.CommitTime;
runSummary.Date = DateTime.UtcNow.ToString("yyyy-MM-dd");
runSummary.Time = DateTime.UtcNow.ToString("HH-mm");
runSummary.BranchName = config.BranchName;
runSummary.TotalOps = config.ItemCount;
runSummary.Concurrency = taskCount;
runSummary.Database = config.Database;
runSummary.Container = config.Container;
runSummary.AccountName = config.EndPoint;
runSummary.pk = config.ResultsPartitionKeyValue;
runSummary.MaxTcpConnectionsPerEndpoint = config.MaxTcpConnectionsPerEndpoint;
runSummary.MaxRequestsPerTcpConnection = config.MaxRequestsPerTcpConnection;
string consistencyLevel = config.ConsistencyLevel;
if (string.IsNullOrWhiteSpace(consistencyLevel))
{
AccountProperties accountProperties = await cosmosClient.ReadAccountAsync();
consistencyLevel = accountProperties.Consistency.DefaultConsistencyLevel.ToString();
}
runSummary.ConsistencyLevel = consistencyLevel;
if (config.PublishResults)
{
runSummary.Diagnostics = CosmosDiagnosticsLogger.GetDiagnostics();
await this.PublishResults(
config,
runSummary,
cosmosClient);
}
return(runSummary);
}
}
/// <summary>
/// Executing benchmarks for V2/V3 cosmosdb SDK.
/// </summary>
/// <returns>a Task object.</returns>
private async Task <RunSummary> ExecuteAsync(BenchmarkConfig config)
{
// V3 SDK client initialization
using (CosmosClient cosmosClient = config.CreateCosmosClient(config.Key))
{
Microsoft.Azure.Cosmos.Database database = cosmosClient.GetDatabase(config.Database);
if (config.CleanupOnStart)
{
await database.DeleteStreamAsync();
}
ContainerResponse containerResponse = await Program.CreatePartitionedContainerAsync(config, cosmosClient);
Container container = containerResponse;
int?currentContainerThroughput = await container.ReadThroughputAsync();
if (!currentContainerThroughput.HasValue)
{
// Container throughput is not configured. It is shared database throughput
ThroughputResponse throughputResponse = await database.ReadThroughputAsync(requestOptions : null);
throw new InvalidOperationException($"Using database {config.Database} with {throughputResponse.Resource.Throughput} RU/s. " +
$"Container {config.Container} must have a configured throughput.");
}
Console.WriteLine($"Using container {config.Container} with {currentContainerThroughput} RU/s");
int taskCount = config.GetTaskCount(currentContainerThroughput.Value);
Console.WriteLine("Starting Inserts with {0} tasks", taskCount);
Console.WriteLine();
string partitionKeyPath = containerResponse.Resource.PartitionKeyPath;
int opsPerTask = config.ItemCount / taskCount;
// TBD: 2 clients SxS some overhead
RunSummary runSummary;
// V2 SDK client initialization
using (Microsoft.Azure.Documents.Client.DocumentClient documentClient = config.CreateDocumentClient(config.Key))
{
Func <IBenchmarkOperation> benchmarkOperationFactory = this.GetBenchmarkFactory(
config,
partitionKeyPath,
cosmosClient,
documentClient);
if (config.DisableCoreSdkLogging)
{
// Do it after client initialization (HACK)
Program.ClearCoreSdkListeners();
}
IExecutionStrategy execution = IExecutionStrategy.StartNew(benchmarkOperationFactory);
runSummary = await execution.ExecuteAsync(config, taskCount, opsPerTask, 0.01);
}
if (config.CleanupOnFinish)
{
Console.WriteLine($"Deleting Database {config.Database}");
await database.DeleteStreamAsync();
}
string consistencyLevel = config.ConsistencyLevel;
if (string.IsNullOrWhiteSpace(consistencyLevel))
{
AccountProperties accountProperties = await cosmosClient.ReadAccountAsync();
consistencyLevel = accountProperties.Consistency.DefaultConsistencyLevel.ToString();
}
runSummary.ConsistencyLevel = consistencyLevel;
if (config.PublishResults)
{
runSummary.Diagnostics = CosmosDiagnosticsLogger.GetDiagnostics();
await this.PublishResults(
config,
runSummary,
cosmosClient);
}
return(runSummary);
}
}
