public void Dispose()
{
_cosmosClient.Dispose();
_cosmosClient = null;
}
public DocumentQueryExpression(bool async, string collectionId, SelectExpression selectExpression, CosmosClient cosmosClient)
{
_async = async;
_collectionId = collectionId;
SelectExpression = selectExpression;
_cosmosClient = cosmosClient;
}
private static IEnumerable <JObject> _Query(
CosmosClient cosmosClient,
QueryContext queryContext,
string collectionId,
SelectExpression selectExpression)
=> new DocumentEnumerable(cosmosClient, queryContext, collectionId, selectExpression);
public SPService(Cosmos cosmos)
{
_cosmosClient = cosmos.Client;
}
public StoryCosmosContext(CosmosSettings cosmosSettings)
{
CosmosClient _client = new CosmosClient(cosmosSettings.Endpoint, cosmosSettings.Key);
StoryContainer = _client.GetContainer(cosmosSettings.DatabaseName, cosmosSettings.StoryContainer);
}
public InProcTestFhirServer(DataStore dataStore, Type startupType)
: base(new Uri("http://localhost/"))
{
var projectDir = GetProjectPath("src", startupType);
var testConfigPath = Path.GetFullPath("testconfiguration.json");
var launchSettings = JObject.Parse(File.ReadAllText(Path.Combine(projectDir, "Properties", "launchSettings.json")));
var configuration = launchSettings["profiles"][dataStore.ToString()]["environmentVariables"].Cast <JProperty>().ToDictionary(p => p.Name, p => p.Value.ToString());
configuration["TestAuthEnvironment:FilePath"] = "testauthenvironment.json";
configuration["FhirServer:Security:Enabled"] = "true";
string inProcEndpoint = "https://inprochost";
configuration["FhirServer:Security:Authentication:Authority"] = inProcEndpoint;
// For local development we will use the Azure Storage Emulator for export.
configuration["FhirServer:Operations:Export:StorageAccountConnection"] = "UseDevelopmentStorage=true";
// enable reindex for testing
configuration["FhirServer:Operations:Reindex:Enabled"] = "true";
// enable import for testing
configuration["FhirServer:Operations:Import:Enabled"] = "true";
configuration["FhirServer:Operations:IntegrationDataStore:StorageAccountConnection"] = "UseDevelopmentStorage=true";
if (startupType.IsDefined(typeof(RequiresIsolatedDatabaseAttribute)))
{
// Alter the configuration so that the server will create a new, isolated database/container.
// Ensure tha the database/container is deleted at the end of the test run (when this instance is disposed)
if (dataStore == DataStore.SqlServer)
{
var connectionStringBuilder = new SqlConnectionStringBuilder(configuration["SqlServer:ConnectionString"]);
var databaseName = connectionStringBuilder.InitialCatalog += "_" + startupType.Name;
configuration["SqlServer:ConnectionString"] = connectionStringBuilder.ToString();
configuration["TaskHosting:Enabled"] = "true";
configuration["TaskHosting:MaxRunningTaskCount"] = "2";
_cleanupDatabase = async() =>
{
connectionStringBuilder.InitialCatalog = "master";
await using var connection = new SqlConnection(connectionStringBuilder.ToString());
await connection.OpenAsync();
SqlConnection.ClearAllPools();
await using SqlCommand command = connection.CreateCommand();
command.CommandTimeout = 600;
command.CommandText = $"DROP DATABASE IF EXISTS {databaseName}";
await command.ExecuteNonQueryAsync();
};
}
else if (dataStore == DataStore.CosmosDb)
{
var collectionId = configuration["FhirServer:CosmosDb:CollectionId"] = $"fhir{ModelInfoProvider.Version}_{startupType.Name}";
_cleanupDatabase = async() =>
{
string ValueOrFallback(string configKey, string fallbackValue)
{
string value = _builtConfiguration[configKey];
return(string.IsNullOrEmpty(value) ? fallbackValue : value);
}
var host = ValueOrFallback("CosmosDb:Host", CosmosDbLocalEmulator.Host);
var key = ValueOrFallback("CosmosDb:Key", CosmosDbLocalEmulator.Key);
var databaseId = ValueOrFallback("CosmosDb:DatabaseId", null) ?? throw new InvalidOperationException("expected CosmosDb:DatabaseId to be set in configuration");
using var client = new CosmosClient(host, key);
Container container = client.GetContainer(databaseId, collectionId);
await container.DeleteContainerAsync();
};
}
}
var builder = WebHost.CreateDefaultBuilder()
.UseContentRoot(Path.GetDirectoryName(startupType.Assembly.Location))
.ConfigureAppConfiguration(configurationBuilder =>
{
configurationBuilder.AddDevelopmentAuthEnvironmentIfConfigured(new ConfigurationBuilder().AddInMemoryCollection(configuration).Build());
configurationBuilder.AddJsonFile(testConfigPath);
configurationBuilder.AddInMemoryCollection(configuration);
})
.UseStartup(startupType)
.ConfigureServices(serviceCollection =>
{
// ensure that HttpClients
// use a message handler for the test server
serviceCollection
.AddHttpClient(Options.DefaultName)
.ConfigurePrimaryHttpMessageHandler(() => new DispatchingHandler(Server.CreateHandler(), inProcEndpoint))
.SetHandlerLifetime(Timeout.InfiniteTimeSpan); // So that it is not disposed after 2 minutes;
serviceCollection.PostConfigure <JwtBearerOptions>(
JwtBearerDefaults.AuthenticationScheme,
options => options.BackchannelHttpHandler = Server.CreateHandler());
});
Server = new TestServer(builder);
_builtConfiguration = Server.Services.GetRequiredService <IConfiguration>();
}
private static async Task BulkLoadContainer()
{
string dbname = null;
string cname = null;
string infile = null;
int maxBatchCount = 0;
int batchSize = 500;
int batchCount = 0;
int taskCount = 0;
long startEpoch = 0;
int exceptions = 0;
try {
dbname = cliArgs[1];
cname = cliArgs[2];
infile = cliArgs[3];
maxBatchCount = Int32.Parse(cliArgs[4]);
Console.WriteLine($"dbname: {dbname}");
Console.WriteLine($"cname: {cname}");
Console.WriteLine($"infile: {infile}");
Console.WriteLine($"mbc: {maxBatchCount}");
cosmosClient = CosmosClientFactory.BulkLoadingClient();
Database database = cosmosClient.GetDatabase(dbname);
Container container = database.GetContainer(cname);
List <Task> tasks = new List <Task>(batchSize);
Console.WriteLine(
$"LoadContainer - db: {dbname}, container: {cname}, infile: {infile}, maxBatchCount: {maxBatchCount}");
startEpoch = EpochMsTime();
using (FileStream fs = File.Open(infile, FileMode.Open, FileAccess.Read, FileShare.ReadWrite)) {
using (BufferedStream bs = new BufferedStream(fs)) {
using (StreamReader sr = new StreamReader(bs)) {
string jsonLine;
while ((jsonLine = sr.ReadLine()) != null)
{
if (batchCount < maxBatchCount)
{
// Parse the JSON line into a "dynamic" object, like a python dict
dynamic jsonDoc = JsonConvert.DeserializeObject <ExpandoObject>(
jsonLine.Trim(), new ExpandoObjectConverter());
// jsonDoc is an instance of class System.Dynamic.ExpandoObject (dynamic)
// Set some additional attributes on the dynamic jsonDoc
DateTimeOffset now = DateTimeOffset.UtcNow;
jsonDoc.doc_epoch = now.ToUnixTimeMilliseconds();
jsonDoc.doc_time = now.ToString("yyyy/MM/dd-HH:mm:ss");
jsonDoc.id = Guid.NewGuid().ToString();
// Add an UpsertItemAsync task to the list of tasks for a batch
tasks.Add(container.UpsertItemAsync(jsonDoc, new PartitionKey(jsonDoc.pk)));
taskCount++;
if (tasks.Count == batchSize)
{
// Display the last jsonDoc in this batch
Console.WriteLine(JsonConvert.SerializeObject(jsonDoc));
batchCount++;
Console.WriteLine($"writing batch {batchCount} ({tasks.Count}) at {EpochMsTime()}");
// execute all of the Tasks in this batch, and await them
await Task.WhenAll(tasks);
// reset the tasks list to an empty list for the next batch
tasks = new List <Task>(batchSize);
}
}
else
{
return;
}
}
}
}
}
if (tasks.Count > 0)
{
// execute the last batch if there are any tasks
batchCount++;
Console.WriteLine($"writing batch {batchCount} ({tasks.Count}) at {EpochMsTime()}");
await Task.WhenAll(tasks);
}
}
catch (Exception e) {
Console.WriteLine($"EXCEPTION: {e.Message}");
exceptions++;
throw;
}
finally {
long finishEpoch = EpochMsTime();
long elapsedMs = finishEpoch - startEpoch;
double elapsedSec = (double)elapsedMs / (double)1000;
double elapsedMin = elapsedSec / (double)60.0;
double docsPerSec = taskCount / elapsedSec;
Console.WriteLine("");
Console.WriteLine("EOJ Totals:");
Console.WriteLine($" Database: {dbname}");
Console.WriteLine($" Container: {cname}");
Console.WriteLine($" Input Filename: {infile}");
Console.WriteLine($" Max Batch Count: {maxBatchCount}");
Console.WriteLine($" BulkLoad startEpoch: {startEpoch}");
Console.WriteLine($" BulkLoad finishEpoch: {finishEpoch}");
Console.WriteLine($" BulkLoad elapsedMs: {elapsedMs}");
Console.WriteLine($" BulkLoad elapsedSec: {elapsedSec}");
Console.WriteLine($" BulkLoad elapsedMin: {elapsedMin}");
Console.WriteLine($" Batch Size: {batchSize}");
Console.WriteLine($" Batch Count: {batchCount}");
Console.WriteLine($" Exceptions: {exceptions}");
Console.WriteLine($" Document/Task count: {taskCount}");
Console.WriteLine($" Document per Second: {docsPerSec}");
Console.WriteLine("");
}
await Task.Delay(0);
}
public async Task InitializeAsync()
{
var fhirStoredProcs = typeof(IStoredProcedure).Assembly
.GetTypes()
.Where(x => !x.IsAbstract && typeof(IStoredProcedure).IsAssignableFrom(x))
.ToArray()
.Select(type => (IStoredProcedure)Activator.CreateInstance(type));
var optionsMonitor = Substitute.For <IOptionsMonitor <CosmosCollectionConfiguration> >();
optionsMonitor.Get(CosmosDb.Constants.CollectionConfigurationName).Returns(_cosmosCollectionConfiguration);
var searchParameterDefinitionManager = new SearchParameterDefinitionManager(ModelInfoProvider.Instance);
await searchParameterDefinitionManager.StartAsync(CancellationToken.None);
_filebasedSearchParameterRegistry = new FilebasedSearchParameterRegistry(searchParameterDefinitionManager, ModelInfoProvider.Instance);
var updaters = new ICollectionUpdater[]
{
new FhirCollectionSettingsUpdater(_cosmosDataStoreConfiguration, optionsMonitor, NullLogger <FhirCollectionSettingsUpdater> .Instance),
new StoredProcedureInstaller(fhirStoredProcs),
new CosmosDbStatusRegistryInitializer(
() => _filebasedSearchParameterRegistry,
new CosmosQueryFactory(
new CosmosResponseProcessor(Substitute.For <IFhirRequestContextAccessor>(), Substitute.For <IMediator>(), NullLogger <CosmosResponseProcessor> .Instance),
NullFhirCosmosQueryLogger.Instance)),
};
var dbLock = new CosmosDbDistributedLockFactory(Substitute.For <Func <IScoped <Container> > >(), NullLogger <CosmosDbDistributedLock> .Instance);
var upgradeManager = new CollectionUpgradeManager(updaters, _cosmosDataStoreConfiguration, optionsMonitor, dbLock, NullLogger <CollectionUpgradeManager> .Instance);
ICosmosClientTestProvider testProvider = new CosmosClientReadWriteTestProvider();
var fhirRequestContextAccessor = new FhirRequestContextAccessor();
var cosmosResponseProcessor = Substitute.For <ICosmosResponseProcessor>();
var responseProcessor = new CosmosResponseProcessor(fhirRequestContextAccessor, Substitute.For <IMediator>(), NullLogger <CosmosResponseProcessor> .Instance);
var handler = new FhirCosmosResponseHandler(() => new NonDisposingScope(_container), _cosmosDataStoreConfiguration, fhirRequestContextAccessor, responseProcessor);
var documentClientInitializer = new FhirCosmosClientInitializer(testProvider, () => new[] { handler }, NullLogger <FhirCosmosClientInitializer> .Instance);
_cosmosClient = documentClientInitializer.CreateCosmosClient(_cosmosDataStoreConfiguration);
var fhirCollectionInitializer = new CollectionInitializer(_cosmosCollectionConfiguration.CollectionId, _cosmosDataStoreConfiguration, _cosmosCollectionConfiguration.InitialCollectionThroughput, upgradeManager, NullLogger <CollectionInitializer> .Instance);
// Cosmos DB emulators throws errors when multiple collections are initialized concurrently.
// Use the semaphore to only allow one initialization at a time.
await CollectionInitializationSemaphore.WaitAsync();
try
{
await documentClientInitializer.InitializeDataStore(_cosmosClient, _cosmosDataStoreConfiguration, new List <ICollectionInitializer> {
fhirCollectionInitializer
});
_container = documentClientInitializer.CreateFhirContainer(_cosmosClient, _cosmosDataStoreConfiguration.DatabaseId, _cosmosCollectionConfiguration.CollectionId);
}
finally
{
CollectionInitializationSemaphore.Release();
}
var cosmosDocumentQueryFactory = new CosmosQueryFactory(cosmosResponseProcessor, NullFhirCosmosQueryLogger.Instance);
var documentClient = new NonDisposingScope(_container);
_fhirDataStore = new CosmosFhirDataStore(
documentClient,
_cosmosDataStoreConfiguration,
optionsMonitor,
cosmosDocumentQueryFactory,
new RetryExceptionPolicyFactory(_cosmosDataStoreConfiguration),
NullLogger <CosmosFhirDataStore> .Instance,
new VersionSpecificModelInfoProvider(),
Options.Create(new CoreFeatureConfiguration()));
_fhirOperationDataStore = new CosmosFhirOperationDataStore(
documentClient,
_cosmosDataStoreConfiguration,
optionsMonitor,
new RetryExceptionPolicyFactory(_cosmosDataStoreConfiguration),
new CosmosQueryFactory(responseProcessor, new NullFhirCosmosQueryLogger()),
NullLogger <CosmosFhirOperationDataStore> .Instance);
_fhirStorageTestHelper = new CosmosDbFhirStorageTestHelper(
_container,
_cosmosDataStoreConfiguration.DatabaseId,
_cosmosCollectionConfiguration.CollectionId);
}
public TransactionDocumentStore(
CosmosClient cosmosClient)
{
_cosmosClient = cosmosClient;
}
public RequestInvokerHandler(CosmosClient client)
{
this.client = client;
this.RequestedClientConsistencyLevel = this.client.ClientOptions.ConsistencyLevel;
}
public void VerifyCosmosConfigurationPropertiesGetUpdated()
{
string endpoint = AccountEndpoint;
string key = MockCosmosUtil.RandomInvalidCorrectlyFormatedAuthKey;
string region = Regions.WestCentralUS;
ConnectionMode connectionMode = ConnectionMode.Gateway;
TimeSpan requestTimeout = TimeSpan.FromDays(1);
int maxConnections = 9001;
string userAgentSuffix = "testSuffix";
RequestHandler preProcessHandler = new TestHandler();
ApiType apiType = ApiType.Sql;
int maxRetryAttemptsOnThrottledRequests = 9999;
TimeSpan maxRetryWaitTime = TimeSpan.FromHours(6);
bool enableTcpConnectionEndpointRediscovery = true;
CosmosSerializationOptions cosmosSerializerOptions = new CosmosSerializationOptions()
{
IgnoreNullValues = true,
PropertyNamingPolicy = CosmosPropertyNamingPolicy.CamelCase,
};
TimeSpan idleTcpConnectionTimeout = new TimeSpan(0, 10, 0);
TimeSpan openTcpConnectionTimeout = new TimeSpan(0, 0, 5);
int maxRequestsPerTcpConnection = 30;
int maxTcpConnectionsPerEndpoint = 65535;
Cosmos.PortReuseMode portReuseMode = Cosmos.PortReuseMode.PrivatePortPool;
IWebProxy webProxy = new TestWebProxy();
Cosmos.ConsistencyLevel consistencyLevel = Cosmos.ConsistencyLevel.ConsistentPrefix;
CosmosClientBuilder cosmosClientBuilder = new CosmosClientBuilder(
accountEndpoint: endpoint,
authKeyOrResourceToken: key);
CosmosClient cosmosClient = cosmosClientBuilder.Build(new MockDocumentClient());
CosmosClientOptions clientOptions = cosmosClient.ClientOptions;
Assert.AreEqual(endpoint, cosmosClient.Endpoint.OriginalString, "AccountEndpoint did not save correctly");
Assert.AreEqual(key, cosmosClient.AccountKey, "AccountKey did not save correctly");
//Verify the default values are different from the new values
Assert.AreNotEqual(region, clientOptions.ApplicationRegion);
Assert.IsNull(clientOptions.ApplicationPreferredRegions);
Assert.AreNotEqual(connectionMode, clientOptions.ConnectionMode);
Assert.AreNotEqual(maxConnections, clientOptions.GatewayModeMaxConnectionLimit);
Assert.AreNotEqual(requestTimeout, clientOptions.RequestTimeout);
Assert.AreNotEqual(userAgentSuffix, clientOptions.ApplicationName);
Assert.AreNotEqual(apiType, clientOptions.ApiType);
Assert.IsFalse(clientOptions.AllowBulkExecution);
Assert.AreEqual(0, clientOptions.CustomHandlers.Count);
Assert.IsNull(clientOptions.SerializerOptions);
Assert.IsNotNull(clientOptions.Serializer);
Assert.IsNull(clientOptions.WebProxy);
Assert.IsFalse(clientOptions.LimitToEndpoint);
Assert.IsTrue(clientOptions.EnableTcpConnectionEndpointRediscovery);
Assert.IsNull(clientOptions.HttpClientFactory);
Assert.AreNotEqual(consistencyLevel, clientOptions.ConsistencyLevel);
Assert.IsFalse(clientOptions.EnablePartitionLevelFailover);
//Verify GetConnectionPolicy returns the correct values for default
ConnectionPolicy policy = clientOptions.GetConnectionPolicy(clientId: 0);
Assert.AreEqual(ConnectionMode.Direct, policy.ConnectionMode);
Assert.AreEqual(Protocol.Tcp, policy.ConnectionProtocol);
Assert.AreEqual(clientOptions.GatewayModeMaxConnectionLimit, policy.MaxConnectionLimit);
Assert.AreEqual(clientOptions.RequestTimeout, policy.RequestTimeout);
Assert.IsNull(policy.IdleTcpConnectionTimeout);
Assert.IsNull(policy.OpenTcpConnectionTimeout);
Assert.IsNull(policy.MaxRequestsPerTcpConnection);
Assert.IsNull(policy.MaxTcpConnectionsPerEndpoint);
Assert.IsTrue(policy.EnableEndpointDiscovery);
Assert.IsTrue(policy.EnableTcpConnectionEndpointRediscovery);
Assert.IsNull(policy.HttpClientFactory);
Assert.AreNotEqual(Cosmos.ConsistencyLevel.Session, clientOptions.ConsistencyLevel);
Assert.IsFalse(policy.EnablePartitionLevelFailover);
cosmosClientBuilder.WithApplicationRegion(region)
.WithConnectionModeGateway(maxConnections, webProxy)
.WithRequestTimeout(requestTimeout)
.WithApplicationName(userAgentSuffix)
.AddCustomHandlers(preProcessHandler)
.WithApiType(apiType)
.WithThrottlingRetryOptions(maxRetryWaitTime, maxRetryAttemptsOnThrottledRequests)
.WithBulkExecution(true)
.WithSerializerOptions(cosmosSerializerOptions)
.WithConsistencyLevel(consistencyLevel)
.WithPartitionLevelFailoverEnabled();
cosmosClient = cosmosClientBuilder.Build(new MockDocumentClient());
clientOptions = cosmosClient.ClientOptions;
//Verify all the values are updated
Assert.AreEqual(region, clientOptions.ApplicationRegion);
Assert.IsNull(clientOptions.ApplicationPreferredRegions);
Assert.AreEqual(connectionMode, clientOptions.ConnectionMode);
Assert.AreEqual(maxConnections, clientOptions.GatewayModeMaxConnectionLimit);
Assert.AreEqual(requestTimeout, clientOptions.RequestTimeout);
Assert.AreEqual(userAgentSuffix, clientOptions.ApplicationName);
Assert.AreEqual(preProcessHandler, clientOptions.CustomHandlers[0]);
Assert.AreEqual(apiType, clientOptions.ApiType);
Assert.AreEqual(maxRetryAttemptsOnThrottledRequests, clientOptions.MaxRetryAttemptsOnRateLimitedRequests);
Assert.AreEqual(maxRetryWaitTime, clientOptions.MaxRetryWaitTimeOnRateLimitedRequests);
Assert.AreEqual(cosmosSerializerOptions.IgnoreNullValues, clientOptions.SerializerOptions.IgnoreNullValues);
Assert.AreEqual(cosmosSerializerOptions.PropertyNamingPolicy, clientOptions.SerializerOptions.PropertyNamingPolicy);
Assert.AreEqual(cosmosSerializerOptions.Indented, clientOptions.SerializerOptions.Indented);
Assert.IsTrue(object.ReferenceEquals(webProxy, clientOptions.WebProxy));
Assert.IsTrue(clientOptions.AllowBulkExecution);
Assert.AreEqual(consistencyLevel, clientOptions.ConsistencyLevel);
Assert.IsTrue(clientOptions.EnablePartitionLevelFailover);
//Verify GetConnectionPolicy returns the correct values
policy = clientOptions.GetConnectionPolicy(clientId: 0);
Assert.AreEqual(region, policy.PreferredLocations[0]);
Assert.AreEqual(ConnectionMode.Gateway, policy.ConnectionMode);
Assert.AreEqual(Protocol.Https, policy.ConnectionProtocol);
Assert.AreEqual(maxConnections, policy.MaxConnectionLimit);
Assert.AreEqual(requestTimeout, policy.RequestTimeout);
Assert.IsTrue(policy.UserAgentSuffix.Contains(userAgentSuffix));
Assert.IsTrue(policy.UseMultipleWriteLocations);
Assert.AreEqual(maxRetryAttemptsOnThrottledRequests, policy.RetryOptions.MaxRetryAttemptsOnThrottledRequests);
Assert.AreEqual((int)maxRetryWaitTime.TotalSeconds, policy.RetryOptions.MaxRetryWaitTimeInSeconds);
Assert.AreEqual((Documents.ConsistencyLevel)consistencyLevel, clientOptions.GetDocumentsConsistencyLevel());
Assert.IsTrue(policy.EnablePartitionLevelFailover);
IReadOnlyList <string> preferredLocations = new List <string>()
{
Regions.AustraliaCentral, Regions.AustraliaCentral2
};
//Verify Direct Mode settings
cosmosClientBuilder = new CosmosClientBuilder(
accountEndpoint: endpoint,
authKeyOrResourceToken: key);
cosmosClientBuilder.WithConnectionModeDirect(
idleTcpConnectionTimeout,
openTcpConnectionTimeout,
maxRequestsPerTcpConnection,
maxTcpConnectionsPerEndpoint,
portReuseMode,
enableTcpConnectionEndpointRediscovery)
.WithApplicationPreferredRegions(preferredLocations);
cosmosClient = cosmosClientBuilder.Build(new MockDocumentClient());
clientOptions = cosmosClient.ClientOptions;
//Verify all the values are updated
Assert.AreEqual(idleTcpConnectionTimeout, clientOptions.IdleTcpConnectionTimeout);
Assert.AreEqual(openTcpConnectionTimeout, clientOptions.OpenTcpConnectionTimeout);
Assert.AreEqual(maxRequestsPerTcpConnection, clientOptions.MaxRequestsPerTcpConnection);
Assert.AreEqual(maxTcpConnectionsPerEndpoint, clientOptions.MaxTcpConnectionsPerEndpoint);
Assert.AreEqual(portReuseMode, clientOptions.PortReuseMode);
Assert.IsTrue(clientOptions.EnableTcpConnectionEndpointRediscovery);
CollectionAssert.AreEqual(preferredLocations.ToArray(), clientOptions.ApplicationPreferredRegions.ToArray());
//Verify GetConnectionPolicy returns the correct values
policy = clientOptions.GetConnectionPolicy(clientId: 0);
Assert.AreEqual(idleTcpConnectionTimeout, policy.IdleTcpConnectionTimeout);
Assert.AreEqual(openTcpConnectionTimeout, policy.OpenTcpConnectionTimeout);
Assert.AreEqual(maxRequestsPerTcpConnection, policy.MaxRequestsPerTcpConnection);
Assert.AreEqual(maxTcpConnectionsPerEndpoint, policy.MaxTcpConnectionsPerEndpoint);
Assert.AreEqual(portReuseMode, policy.PortReuseMode);
Assert.IsTrue(policy.EnableTcpConnectionEndpointRediscovery);
CollectionAssert.AreEqual(preferredLocations.ToArray(), policy.PreferredLocations.ToArray());
}
public RunProcessor(IFailureAnalyzer failureAnalyzer, ILogger <RunProcessor> logger, IMemoryCache cache, SecretClient secretClient, CosmosClient cosmosClient, VssConnection vssConnection)
{
this.failureAnalyzer = failureAnalyzer;
this.logger = logger;
this.cache = cache;
this.secretClient = secretClient;
this.cosmosClient = cosmosClient;
this.vssConnection = vssConnection;
}
public async Task ReadMany404ExceptionTest()
{
Database database = await this.cosmosClient.CreateDatabaseAsync(Guid.NewGuid().ToString());
Container container = await database.CreateContainerAsync(Guid.NewGuid().ToString(), "/pk");
for (int i = 0; i < 5; i++)
{
await container.CreateItemAsync(
ToDoActivity.CreateRandomToDoActivity("pk" + i, i.ToString()));
}
List <(string, PartitionKey)> itemList = new List <(string, PartitionKey)>();
for (int i = 0; i < 5; i++)
{
itemList.Add((i.ToString(), new PartitionKey("pk" + i)));
}
// 429 test
//List<Task> tasks = new List<Task>();
//ConcurrentQueue<ResponseMessage> failedRequests = new ConcurrentQueue<ResponseMessage>();
//for (int i = 0; i < 500; i++)
//{
// tasks.Add(Task.Run(async () =>
// {
// ResponseMessage responseMessage = await container.ReadManyItemsStreamAsync(itemList);
// if (!responseMessage.IsSuccessStatusCode)
// {
// failedRequests.Enqueue(responseMessage);
// Assert.AreEqual(responseMessage.StatusCode, HttpStatusCode.TooManyRequests);
// }
// }));
//}
//await Task.WhenAll(tasks);
//Assert.IsTrue(failedRequests.Count > 0);
await container.ReadManyItemsAsync <ToDoActivity>(itemList); // Warm up caches
using (CosmosClient cosmosClient = TestCommon.CreateCosmosClient())
{
Container newContainer = cosmosClient.GetContainer(database.Id, container.Id);
await newContainer.DeleteContainerAsync();
}
using (ResponseMessage responseMessage = await container.ReadManyItemsStreamAsync(itemList))
{
Assert.AreEqual(responseMessage.StatusCode, HttpStatusCode.NotFound);
}
try
{
await container.ReadManyItemsAsync <ToDoActivity>(itemList);
Assert.Fail("Typed API should throw");
}
catch (CosmosException ex)
{
Assert.AreEqual(ex.Error.Code, "NotFound");
}
await database.DeleteAsync();
}
private async Task StartDemo()
{
Console.WriteLine("Starting Cosmos DB SQL API Demo!");
//Create a new demo database
string databaseName = "demoDB_" + Guid.NewGuid().ToString().Substring(0, 5);
this.SendMessageToConsoleAndWait($"Creating database {databaseName}...");
this.client = new CosmosClient(EndpointUri, Key);
this.database = await this.client.CreateDatabaseIfNotExistsAsync(databaseName);
//Create a new demo collection inside the demo database.
//This creates a collection with a reserved throughput. You can customize the options using a ContainerProperties object
//This operation has pricing implications.
string containerName = "collection_" + Guid.NewGuid().ToString().Substring(0, 5);
this.SendMessageToConsoleAndWait($"Creating collection demo{containerName}...");
this.container = await this.database.CreateContainerIfNotExistsAsync(containerName, "/LastName");
//Create some documents in the collection
Person person1 = new Person
{
Id = "Person.1",
FirstName = "Santiago",
LastName = "Fernandez",
Devices = new Device[]
{
new Device {
OperatingSystem = "iOS", CameraMegaPixels = 7,
Ram = 16, Usage = "Personal"
},
new Device {
OperatingSystem = "Android", CameraMegaPixels = 12,
Ram = 64, Usage = "Work"
}
},
Gender = "Male",
Address = new Address
{
City = "Seville",
Country = "Spain",
PostalCode = "28973",
Street = "Diagonal",
State = "Andalucia"
},
IsRegistered = true
};
await this.CreateDocumentIfNotExistsAsync(databaseName, containerName, person1);
Person person2 = new Person
{
Id = "Person.2",
FirstName = "Agatha",
LastName = "Smith",
Devices = new Device[]
{
new Device {
OperatingSystem = "iOS", CameraMegaPixels = 12,
Ram = 32, Usage = "Work"
},
new Device {
OperatingSystem = "Windows", CameraMegaPixels = 12,
Ram = 64, Usage = "Personal"
}
},
Gender = "Female",
Address = new Address
{
City = "Laguna Beach",
Country = "United States",
PostalCode = "12345",
Street = "Main",
State = "CA"
},
IsRegistered = true
};
await this.CreateDocumentIfNotExistsAsync(databaseName, containerName, person2);
//Make some queries to the collection
this.SendMessageToConsoleAndWait($"Getting documents from the collection{containerName}...");
//Find documents using LINQ
IQueryable <Person> queryablePeople = this.container.GetItemLinqQueryable <Person>(true)
.Where(p => p.Gender == "Male");
System.Console.WriteLine("Running LINQ query for finding men...");
foreach (Person foundPerson in queryablePeople)
{
System.Console.WriteLine($"\tPerson: {foundPerson}");
}
//Find documents using SQL
var sqlQuery = "SELECT * FROM Person WHERE Person.Gender = 'Female'";
QueryDefinition queryDefinition = new QueryDefinition(sqlQuery);
FeedIterator <Person> peopleResultSetIterator = this.container.GetItemQueryIterator <Person>(queryDefinition);
System.Console.WriteLine("Running SQL query for finding women...");
while (peopleResultSetIterator.HasMoreResults)
{
FeedResponse <Person> currentResultSet = await peopleResultSetIterator.ReadNextAsync();
foreach (Person foundPerson in currentResultSet)
{
System.Console.WriteLine($"\tPerson: {foundPerson}");
}
}
Console.WriteLine("Press any key to continue...");
Console.ReadKey();
//Update documents in a collection
this.SendMessageToConsoleAndWait($"Updating documents in the collection { containerName}...");
person2.FirstName = "Mathew";
person2.Gender = "Male";
await this.container.UpsertItemAsync(person2);
this.SendMessageToConsoleAndWait($"Document modified {person2}");
//Delete a single document from the collection
this.SendMessageToConsoleAndWait($"Deleting documents from the collection { containerName}...");
PartitionKey partitionKey = new PartitionKey(person1.LastName);
await this.container.DeleteItemAsync <Person>(person1.Id, partitionKey);
this.SendMessageToConsoleAndWait($"Document deleted {person1}");
//Delete created demo database and all its children elements
this.SendMessageToConsoleAndWait("Cleaning-up your Cosmos DB account...");
await this.database.DeleteAsync();
}
public async Task ShouldReturnNotModifiedAfterCyclingOnAllRanges()
{
// Setting mock to have 3 ranges, this test will get a 304 on all 3 ranges, do 3 backend requests, and return a 304
MultiRangeMockDocumentClient documentClient = new MultiRangeMockDocumentClient();
CosmosClient client = MockCosmosUtil.CreateMockCosmosClient();
Mock <CosmosClientContext> mockContext = new Mock <CosmosClientContext>();
mockContext.Setup(x => x.ClientOptions).Returns(MockCosmosUtil.GetDefaultConfiguration());
mockContext.Setup(x => x.DocumentClient).Returns(documentClient);
mockContext.Setup(x => x.SerializerCore).Returns(MockCosmosUtil.Serializer);
mockContext.Setup(x => x.Client).Returns(client);
mockContext.Setup(x => x.CreateLink(It.IsAny <string>(), It.IsAny <string>(), It.IsAny <string>())).Returns(new Uri("/dbs/test/colls/test", UriKind.Relative));
ResponseMessage firstResponse = new ResponseMessage(HttpStatusCode.NotModified);
firstResponse.Headers.ETag = "FirstContinuation";
ResponseMessage secondResponse = new ResponseMessage(HttpStatusCode.NotModified);
secondResponse.Headers.ETag = "SecondContinuation";
ResponseMessage thirdResponse = new ResponseMessage(HttpStatusCode.NotModified);
thirdResponse.Headers.ETag = "ThirdContinuation";
mockContext.SetupSequence(x => x.ProcessResourceOperationAsync <ResponseMessage>(
It.IsAny <Uri>(),
It.IsAny <Documents.ResourceType>(),
It.IsAny <Documents.OperationType>(),
It.IsAny <RequestOptions>(),
It.IsAny <ContainerCore>(),
It.IsAny <PartitionKey?>(),
It.IsAny <Stream>(),
It.IsAny <Action <RequestMessage> >(),
It.IsAny <Func <ResponseMessage, ResponseMessage> >(),
It.IsAny <CosmosDiagnosticsContext>(),
It.IsAny <CancellationToken>()))
.Returns(Task.FromResult(firstResponse))
.Returns(Task.FromResult(secondResponse))
.Returns(Task.FromResult(thirdResponse));
DatabaseCore databaseCore = new DatabaseCore(mockContext.Object, "mydb");
StandByFeedIteratorCore iterator = new StandByFeedIteratorCore(
mockContext.Object, new ContainerCore(mockContext.Object, databaseCore, "myColl"), null, 10, new ChangeFeedRequestOptions());
ResponseMessage firstRequest = await iterator.ReadNextAsync();
Assert.IsTrue(firstRequest.Headers.ContinuationToken.Contains(firstResponse.Headers.ETag), "Response should contain the first continuation");
Assert.IsTrue(firstRequest.Headers.ContinuationToken.Contains(secondResponse.Headers.ETag), "Response should contain the second continuation");
Assert.IsTrue(firstRequest.Headers.ContinuationToken.Contains(thirdResponse.Headers.ETag), "Response should contain the third continuation");
Assert.AreEqual(HttpStatusCode.NotModified, firstRequest.StatusCode);
mockContext.Verify(x => x.ProcessResourceOperationAsync <ResponseMessage>(
It.IsAny <Uri>(),
It.IsAny <Documents.ResourceType>(),
It.IsAny <Documents.OperationType>(),
It.IsAny <RequestOptions>(),
It.IsAny <ContainerCore>(),
It.IsAny <PartitionKey?>(),
It.IsAny <Stream>(),
It.IsAny <Action <RequestMessage> >(),
It.IsAny <Func <ResponseMessage, ResponseMessage> >(),
It.IsAny <CosmosDiagnosticsContext>(),
It.IsAny <CancellationToken>()), Times.Exactly(3));
}
static async Task Main(string[] args)
{
string DatabaseName = "bulk-tutorial";
string ContainerName = "items";
string Account;
string Key;
int ItemsToInsert = 300000;
if (ParseCommandLine(args, out Account, out Key, out DatabaseName, out ContainerName, out ItemsToInsert))
{
string EndpointUrl = $"https://{Account}.documents.azure.com:443/";
string AuthorizationKey = Key;
CosmosClient cosmosClient = new CosmosClient(EndpointUrl, AuthorizationKey, new CosmosClientOptions()
{
AllowBulkExecution = true
});
// Create with a throughput of 50000 RU/s
// Indexing Policy to exclude all attributes to maximize RU/s usage
Console.WriteLine("This sample app will create a 50000 RU/s container, press any key to continue.");
Console.ReadKey();
// <Initialize>
Database database = await cosmosClient.CreateDatabaseIfNotExistsAsync(DatabaseName);
await database.DefineContainer(ContainerName, "/pk")
.WithIndexingPolicy()
.WithIndexingMode(IndexingMode.Consistent)
.WithIncludedPaths()
.Attach()
.WithExcludedPaths()
.Path("/*")
.Attach()
.Attach()
.CreateAsync(50000);
// </Initialize>
try
{
// Prepare items for insertion
Console.WriteLine($"Preparing {ItemsToInsert} items to insert...");
// <Operations>
Dictionary <PartitionKey, Stream> itemsToInsert = new Dictionary <PartitionKey, Stream>(ItemsToInsert);
foreach (Item item in Program.GetItemsToInsert(ItemsToInsert))
{
MemoryStream stream = new MemoryStream();
await JsonSerializer.SerializeAsync(stream, item);
itemsToInsert.Add(new PartitionKey(item.pk), stream);
}
// </Operations>
// Create the list of Tasks
Console.WriteLine($"Starting...");
Stopwatch stopwatch = Stopwatch.StartNew();
// <ConcurrentTasks>
Container container = database.GetContainer(ContainerName);
List <Task> tasks = new List <Task>(ItemsToInsert);
foreach (KeyValuePair <PartitionKey, Stream> item in itemsToInsert)
{
tasks.Add(container.CreateItemStreamAsync(item.Value, item.Key)
.ContinueWith((Task <ResponseMessage> task) =>
{
using (ResponseMessage response = task.Result)
{
if (!response.IsSuccessStatusCode)
{
Console.WriteLine($"Received {response.StatusCode} ({response.ErrorMessage}) status code for operation {EndpointUrl}.");
}
}
}));
}
// Wait until all are done
await Task.WhenAll(tasks);
// </ConcurrentTasks>
stopwatch.Stop();
Console.WriteLine($"Finished in writing {ItemsToInsert} items in {stopwatch.Elapsed}.");
}
catch (Exception ex)
{
Console.WriteLine(ex);
}
finally
{
Console.WriteLine("This sample app will delete the database, press any key to continue.");
Console.ReadKey();
Console.WriteLine("Cleaning up resources...");
await database.DeleteAsync();
}
}
}
public async Task ContinuationTokenIsNotUpdatedOnFails()
{
MultiRangeMockDocumentClient documentClient = new MultiRangeMockDocumentClient();
CosmosClient client = MockCosmosUtil.CreateMockCosmosClient();
Mock <CosmosClientContext> mockContext = new Mock <CosmosClientContext>();
mockContext.Setup(x => x.ClientOptions).Returns(MockCosmosUtil.GetDefaultConfiguration());
mockContext.Setup(x => x.DocumentClient).Returns(documentClient);
mockContext.Setup(x => x.SerializerCore).Returns(MockCosmosUtil.Serializer);
mockContext.Setup(x => x.Client).Returns(client);
mockContext.Setup(x => x.CreateLink(It.IsAny <string>(), It.IsAny <string>(), It.IsAny <string>())).Returns(new Uri("/dbs/test/colls/test", UriKind.Relative));
ResponseMessage firstResponse = new ResponseMessage(HttpStatusCode.NotModified);
firstResponse.Headers.ETag = "FirstContinuation";
ResponseMessage secondResponse = new ResponseMessage(
statusCode: HttpStatusCode.NotFound,
requestMessage: null,
headers: new Headers()
{
ETag = "ShouldNotContainThis"
},
cosmosException: CosmosExceptionFactory.CreateNotFoundException("something"),
diagnostics: new CosmosDiagnosticsContextCore());
mockContext.SetupSequence(x => x.ProcessResourceOperationAsync <ResponseMessage>(
It.IsAny <Uri>(),
It.IsAny <Documents.ResourceType>(),
It.IsAny <Documents.OperationType>(),
It.IsAny <RequestOptions>(),
It.IsAny <ContainerCore>(),
It.IsAny <PartitionKey?>(),
It.IsAny <Stream>(),
It.IsAny <Action <RequestMessage> >(),
It.IsAny <Func <ResponseMessage, ResponseMessage> >(),
It.IsAny <CosmosDiagnosticsContext>(),
It.IsAny <CancellationToken>()))
.Returns(Task.FromResult(firstResponse))
.Returns(Task.FromResult(secondResponse));
DatabaseCore databaseCore = new DatabaseCore(mockContext.Object, "mydb");
StandByFeedIteratorCore iterator = new StandByFeedIteratorCore(
mockContext.Object, new ContainerCore(mockContext.Object, databaseCore, "myColl"), null, 10, new ChangeFeedRequestOptions());
ResponseMessage firstRequest = await iterator.ReadNextAsync();
Assert.IsTrue(firstRequest.Headers.ContinuationToken.Contains(firstResponse.Headers.ETag), "Response should contain the first continuation");
Assert.IsTrue(!firstRequest.Headers.ContinuationToken.Contains(secondResponse.Headers.ETag), "Response should not contain the second continuation");
Assert.AreEqual(HttpStatusCode.NotFound, firstRequest.StatusCode);
mockContext.Verify(x => x.ProcessResourceOperationAsync <ResponseMessage>(
It.IsAny <Uri>(),
It.IsAny <Documents.ResourceType>(),
It.IsAny <Documents.OperationType>(),
It.IsAny <RequestOptions>(),
It.IsAny <ContainerCore>(),
It.IsAny <PartitionKey?>(),
It.IsAny <Stream>(),
It.IsAny <Action <RequestMessage> >(),
It.IsAny <Func <ResponseMessage, ResponseMessage> >(),
It.IsAny <CosmosDiagnosticsContext>(),
It.IsAny <CancellationToken>()), Times.Exactly(2));
}
// Use Dependency Injection to inject the HttpClientFactory service and Cosmos DB client that were configured in Startup.cs.
public Functions(IHttpClientFactory httpClientFactory, CosmosClient cosmosClient)
{
_httpClientFactory = httpClientFactory;
_cosmosClient = cosmosClient;
}
