示例#1
0
 public void Dispose()
 {
     _cosmosClient.Dispose();
     _cosmosClient = null;
 }
示例#2
0
 public DocumentQueryExpression(bool async, string collectionId, SelectExpression selectExpression, CosmosClient cosmosClient)
 {
     _async           = async;
     _collectionId    = collectionId;
     SelectExpression = selectExpression;
     _cosmosClient    = cosmosClient;
 }
示例#3
0
 private static IEnumerable <JObject> _Query(
     CosmosClient cosmosClient,
     QueryContext queryContext,
     string collectionId,
     SelectExpression selectExpression)
 => new DocumentEnumerable(cosmosClient, queryContext, collectionId, selectExpression);
示例#4
0
 public SPService(Cosmos cosmos)
 {
     _cosmosClient = cosmos.Client;
 }
        public StoryCosmosContext(CosmosSettings cosmosSettings)
        {
            CosmosClient _client = new CosmosClient(cosmosSettings.Endpoint, cosmosSettings.Key);

            StoryContainer = _client.GetContainer(cosmosSettings.DatabaseName, cosmosSettings.StoryContainer);
        }
        public InProcTestFhirServer(DataStore dataStore, Type startupType)
            : base(new Uri("http://localhost/"))
        {
            var projectDir     = GetProjectPath("src", startupType);
            var testConfigPath = Path.GetFullPath("testconfiguration.json");

            var launchSettings = JObject.Parse(File.ReadAllText(Path.Combine(projectDir, "Properties", "launchSettings.json")));

            var configuration = launchSettings["profiles"][dataStore.ToString()]["environmentVariables"].Cast <JProperty>().ToDictionary(p => p.Name, p => p.Value.ToString());

            configuration["TestAuthEnvironment:FilePath"] = "testauthenvironment.json";
            configuration["FhirServer:Security:Enabled"]  = "true";

            string inProcEndpoint = "https://inprochost";

            configuration["FhirServer:Security:Authentication:Authority"] = inProcEndpoint;

            // For local development we will use the Azure Storage Emulator for export.
            configuration["FhirServer:Operations:Export:StorageAccountConnection"] = "UseDevelopmentStorage=true";

            // enable reindex for testing
            configuration["FhirServer:Operations:Reindex:Enabled"] = "true";

            // enable import for testing
            configuration["FhirServer:Operations:Import:Enabled"] = "true";
            configuration["FhirServer:Operations:IntegrationDataStore:StorageAccountConnection"] = "UseDevelopmentStorage=true";

            if (startupType.IsDefined(typeof(RequiresIsolatedDatabaseAttribute)))
            {
                // Alter the configuration so that the server will create a new, isolated database/container.
                // Ensure tha the database/container is deleted at the end of the test run (when this instance is disposed)

                if (dataStore == DataStore.SqlServer)
                {
                    var connectionStringBuilder = new SqlConnectionStringBuilder(configuration["SqlServer:ConnectionString"]);
                    var databaseName            = connectionStringBuilder.InitialCatalog += "_" + startupType.Name;
                    configuration["SqlServer:ConnectionString"]      = connectionStringBuilder.ToString();
                    configuration["TaskHosting:Enabled"]             = "true";
                    configuration["TaskHosting:MaxRunningTaskCount"] = "2";

                    _cleanupDatabase = async() =>
                    {
                        connectionStringBuilder.InitialCatalog = "master";
                        await using var connection             = new SqlConnection(connectionStringBuilder.ToString());

                        await connection.OpenAsync();

                        SqlConnection.ClearAllPools();

                        await using SqlCommand command = connection.CreateCommand();
                        command.CommandTimeout         = 600;
                        command.CommandText            = $"DROP DATABASE IF EXISTS {databaseName}";
                        await command.ExecuteNonQueryAsync();
                    };
                }
                else if (dataStore == DataStore.CosmosDb)
                {
                    var collectionId = configuration["FhirServer:CosmosDb:CollectionId"] = $"fhir{ModelInfoProvider.Version}_{startupType.Name}";

                    _cleanupDatabase = async() =>
                    {
                        string ValueOrFallback(string configKey, string fallbackValue)
                        {
                            string value = _builtConfiguration[configKey];

                            return(string.IsNullOrEmpty(value) ? fallbackValue : value);
                        }

                        var host       = ValueOrFallback("CosmosDb:Host", CosmosDbLocalEmulator.Host);
                        var key        = ValueOrFallback("CosmosDb:Key", CosmosDbLocalEmulator.Key);
                        var databaseId = ValueOrFallback("CosmosDb:DatabaseId", null) ?? throw new InvalidOperationException("expected CosmosDb:DatabaseId to be set in configuration");

                        using var client = new CosmosClient(host, key);
                        Container container = client.GetContainer(databaseId, collectionId);
                        await container.DeleteContainerAsync();
                    };
                }
            }

            var builder = WebHost.CreateDefaultBuilder()
                          .UseContentRoot(Path.GetDirectoryName(startupType.Assembly.Location))
                          .ConfigureAppConfiguration(configurationBuilder =>
            {
                configurationBuilder.AddDevelopmentAuthEnvironmentIfConfigured(new ConfigurationBuilder().AddInMemoryCollection(configuration).Build());
                configurationBuilder.AddJsonFile(testConfigPath);
                configurationBuilder.AddInMemoryCollection(configuration);
            })
                          .UseStartup(startupType)
                          .ConfigureServices(serviceCollection =>
            {
                // ensure that HttpClients
                // use a message handler for the test server
                serviceCollection
                .AddHttpClient(Options.DefaultName)
                .ConfigurePrimaryHttpMessageHandler(() => new DispatchingHandler(Server.CreateHandler(), inProcEndpoint))
                .SetHandlerLifetime(Timeout.InfiniteTimeSpan);         // So that it is not disposed after 2 minutes;

                serviceCollection.PostConfigure <JwtBearerOptions>(
                    JwtBearerDefaults.AuthenticationScheme,
                    options => options.BackchannelHttpHandler = Server.CreateHandler());
            });

            Server = new TestServer(builder);

            _builtConfiguration = Server.Services.GetRequiredService <IConfiguration>();
        }
示例#7
0
        private static async Task BulkLoadContainer()
        {
            string dbname        = null;
            string cname         = null;
            string infile        = null;
            int    maxBatchCount = 0;
            int    batchSize     = 500;
            int    batchCount    = 0;
            int    taskCount     = 0;
            long   startEpoch    = 0;
            int    exceptions    = 0;

            try {
                dbname        = cliArgs[1];
                cname         = cliArgs[2];
                infile        = cliArgs[3];
                maxBatchCount = Int32.Parse(cliArgs[4]);

                Console.WriteLine($"dbname:   {dbname}");
                Console.WriteLine($"cname:    {cname}");
                Console.WriteLine($"infile:   {infile}");
                Console.WriteLine($"mbc:      {maxBatchCount}");

                cosmosClient = CosmosClientFactory.BulkLoadingClient();
                Database    database  = cosmosClient.GetDatabase(dbname);
                Container   container = database.GetContainer(cname);
                List <Task> tasks     = new List <Task>(batchSize);
                Console.WriteLine(
                    $"LoadContainer - db: {dbname}, container: {cname}, infile: {infile}, maxBatchCount: {maxBatchCount}");
                startEpoch = EpochMsTime();

                using (FileStream fs = File.Open(infile, FileMode.Open, FileAccess.Read, FileShare.ReadWrite)) {
                    using (BufferedStream bs = new BufferedStream(fs)) {
                        using (StreamReader sr = new StreamReader(bs)) {
                            string jsonLine;
                            while ((jsonLine = sr.ReadLine()) != null)
                            {
                                if (batchCount < maxBatchCount)
                                {
                                    // Parse the JSON line into a "dynamic" object, like a python dict
                                    dynamic jsonDoc = JsonConvert.DeserializeObject <ExpandoObject>(
                                        jsonLine.Trim(), new ExpandoObjectConverter());
                                    // jsonDoc is an instance of class System.Dynamic.ExpandoObject (dynamic)

                                    // Set some additional attributes on the dynamic jsonDoc
                                    DateTimeOffset now = DateTimeOffset.UtcNow;
                                    jsonDoc.doc_epoch = now.ToUnixTimeMilliseconds();
                                    jsonDoc.doc_time  = now.ToString("yyyy/MM/dd-HH:mm:ss");
                                    jsonDoc.id        = Guid.NewGuid().ToString();

                                    // Add an UpsertItemAsync task to the list of tasks for a batch
                                    tasks.Add(container.UpsertItemAsync(jsonDoc, new PartitionKey(jsonDoc.pk)));
                                    taskCount++;

                                    if (tasks.Count == batchSize)
                                    {
                                        // Display the last jsonDoc in this batch
                                        Console.WriteLine(JsonConvert.SerializeObject(jsonDoc));

                                        batchCount++;
                                        Console.WriteLine($"writing batch {batchCount} ({tasks.Count}) at {EpochMsTime()}");

                                        // execute all of the Tasks in this batch, and await them
                                        await Task.WhenAll(tasks);

                                        // reset the tasks list to an empty list for the next batch
                                        tasks = new List <Task>(batchSize);
                                    }
                                }
                                else
                                {
                                    return;
                                }
                            }
                        }
                    }
                }
                if (tasks.Count > 0)
                {
                    // execute the last batch if there are any tasks
                    batchCount++;
                    Console.WriteLine($"writing batch {batchCount} ({tasks.Count}) at {EpochMsTime()}");
                    await Task.WhenAll(tasks);
                }
            }
            catch (Exception e) {
                Console.WriteLine($"EXCEPTION: {e.Message}");
                exceptions++;
                throw;
            }
            finally {
                long   finishEpoch = EpochMsTime();
                long   elapsedMs   = finishEpoch - startEpoch;
                double elapsedSec  = (double)elapsedMs / (double)1000;
                double elapsedMin  = elapsedSec / (double)60.0;
                double docsPerSec  = taskCount / elapsedSec;
                Console.WriteLine("");
                Console.WriteLine("EOJ Totals:");
                Console.WriteLine($"  Database:             {dbname}");
                Console.WriteLine($"  Container:            {cname}");
                Console.WriteLine($"  Input Filename:       {infile}");
                Console.WriteLine($"  Max Batch Count:      {maxBatchCount}");
                Console.WriteLine($"  BulkLoad startEpoch:  {startEpoch}");
                Console.WriteLine($"  BulkLoad finishEpoch: {finishEpoch}");
                Console.WriteLine($"  BulkLoad elapsedMs:   {elapsedMs}");
                Console.WriteLine($"  BulkLoad elapsedSec:  {elapsedSec}");
                Console.WriteLine($"  BulkLoad elapsedMin:  {elapsedMin}");
                Console.WriteLine($"  Batch Size:           {batchSize}");
                Console.WriteLine($"  Batch Count:          {batchCount}");
                Console.WriteLine($"  Exceptions:           {exceptions}");
                Console.WriteLine($"  Document/Task count:  {taskCount}");
                Console.WriteLine($"  Document per Second:  {docsPerSec}");
                Console.WriteLine("");
            }
            await Task.Delay(0);
        }
        public async Task InitializeAsync()
        {
            var fhirStoredProcs = typeof(IStoredProcedure).Assembly
                                  .GetTypes()
                                  .Where(x => !x.IsAbstract && typeof(IStoredProcedure).IsAssignableFrom(x))
                                  .ToArray()
                                  .Select(type => (IStoredProcedure)Activator.CreateInstance(type));

            var optionsMonitor = Substitute.For <IOptionsMonitor <CosmosCollectionConfiguration> >();

            optionsMonitor.Get(CosmosDb.Constants.CollectionConfigurationName).Returns(_cosmosCollectionConfiguration);

            var searchParameterDefinitionManager = new SearchParameterDefinitionManager(ModelInfoProvider.Instance);
            await searchParameterDefinitionManager.StartAsync(CancellationToken.None);

            _filebasedSearchParameterRegistry = new FilebasedSearchParameterRegistry(searchParameterDefinitionManager, ModelInfoProvider.Instance);

            var updaters = new ICollectionUpdater[]
            {
                new FhirCollectionSettingsUpdater(_cosmosDataStoreConfiguration, optionsMonitor, NullLogger <FhirCollectionSettingsUpdater> .Instance),
                new StoredProcedureInstaller(fhirStoredProcs),
                new CosmosDbStatusRegistryInitializer(
                    () => _filebasedSearchParameterRegistry,
                    new CosmosQueryFactory(
                        new CosmosResponseProcessor(Substitute.For <IFhirRequestContextAccessor>(), Substitute.For <IMediator>(), NullLogger <CosmosResponseProcessor> .Instance),
                        NullFhirCosmosQueryLogger.Instance)),
            };

            var dbLock = new CosmosDbDistributedLockFactory(Substitute.For <Func <IScoped <Container> > >(), NullLogger <CosmosDbDistributedLock> .Instance);

            var upgradeManager = new CollectionUpgradeManager(updaters, _cosmosDataStoreConfiguration, optionsMonitor, dbLock, NullLogger <CollectionUpgradeManager> .Instance);
            ICosmosClientTestProvider testProvider = new CosmosClientReadWriteTestProvider();

            var fhirRequestContextAccessor = new FhirRequestContextAccessor();
            var cosmosResponseProcessor    = Substitute.For <ICosmosResponseProcessor>();

            var responseProcessor         = new CosmosResponseProcessor(fhirRequestContextAccessor, Substitute.For <IMediator>(), NullLogger <CosmosResponseProcessor> .Instance);
            var handler                   = new FhirCosmosResponseHandler(() => new NonDisposingScope(_container), _cosmosDataStoreConfiguration, fhirRequestContextAccessor, responseProcessor);
            var documentClientInitializer = new FhirCosmosClientInitializer(testProvider, () => new[] { handler }, NullLogger <FhirCosmosClientInitializer> .Instance);

            _cosmosClient = documentClientInitializer.CreateCosmosClient(_cosmosDataStoreConfiguration);
            var fhirCollectionInitializer = new CollectionInitializer(_cosmosCollectionConfiguration.CollectionId, _cosmosDataStoreConfiguration, _cosmosCollectionConfiguration.InitialCollectionThroughput, upgradeManager, NullLogger <CollectionInitializer> .Instance);

            // Cosmos DB emulators throws errors when multiple collections are initialized concurrently.
            // Use the semaphore to only allow one initialization at a time.
            await CollectionInitializationSemaphore.WaitAsync();

            try
            {
                await documentClientInitializer.InitializeDataStore(_cosmosClient, _cosmosDataStoreConfiguration, new List <ICollectionInitializer> {
                    fhirCollectionInitializer
                });

                _container = documentClientInitializer.CreateFhirContainer(_cosmosClient, _cosmosDataStoreConfiguration.DatabaseId, _cosmosCollectionConfiguration.CollectionId);
            }
            finally
            {
                CollectionInitializationSemaphore.Release();
            }

            var cosmosDocumentQueryFactory = new CosmosQueryFactory(cosmosResponseProcessor, NullFhirCosmosQueryLogger.Instance);

            var documentClient = new NonDisposingScope(_container);

            _fhirDataStore = new CosmosFhirDataStore(
                documentClient,
                _cosmosDataStoreConfiguration,
                optionsMonitor,
                cosmosDocumentQueryFactory,
                new RetryExceptionPolicyFactory(_cosmosDataStoreConfiguration),
                NullLogger <CosmosFhirDataStore> .Instance,
                new VersionSpecificModelInfoProvider(),
                Options.Create(new CoreFeatureConfiguration()));

            _fhirOperationDataStore = new CosmosFhirOperationDataStore(
                documentClient,
                _cosmosDataStoreConfiguration,
                optionsMonitor,
                new RetryExceptionPolicyFactory(_cosmosDataStoreConfiguration),
                new CosmosQueryFactory(responseProcessor, new NullFhirCosmosQueryLogger()),
                NullLogger <CosmosFhirOperationDataStore> .Instance);

            _fhirStorageTestHelper = new CosmosDbFhirStorageTestHelper(
                _container,
                _cosmosDataStoreConfiguration.DatabaseId,
                _cosmosCollectionConfiguration.CollectionId);
        }
 public TransactionDocumentStore(
     CosmosClient cosmosClient)
 {
     _cosmosClient = cosmosClient;
 }
示例#10
0
 public RequestInvokerHandler(CosmosClient client)
 {
     this.client = client;
     this.RequestedClientConsistencyLevel = this.client.ClientOptions.ConsistencyLevel;
 }
示例#11
0
        public void VerifyCosmosConfigurationPropertiesGetUpdated()
        {
            string                     endpoint          = AccountEndpoint;
            string                     key               = MockCosmosUtil.RandomInvalidCorrectlyFormatedAuthKey;
            string                     region            = Regions.WestCentralUS;
            ConnectionMode             connectionMode    = ConnectionMode.Gateway;
            TimeSpan                   requestTimeout    = TimeSpan.FromDays(1);
            int                        maxConnections    = 9001;
            string                     userAgentSuffix   = "testSuffix";
            RequestHandler             preProcessHandler = new TestHandler();
            ApiType                    apiType           = ApiType.Sql;
            int                        maxRetryAttemptsOnThrottledRequests = 9999;
            TimeSpan                   maxRetryWaitTime = TimeSpan.FromHours(6);
            bool                       enableTcpConnectionEndpointRediscovery = true;
            CosmosSerializationOptions cosmosSerializerOptions = new CosmosSerializationOptions()
            {
                IgnoreNullValues     = true,
                PropertyNamingPolicy = CosmosPropertyNamingPolicy.CamelCase,
            };
            TimeSpan idleTcpConnectionTimeout     = new TimeSpan(0, 10, 0);
            TimeSpan openTcpConnectionTimeout     = new TimeSpan(0, 0, 5);
            int      maxRequestsPerTcpConnection  = 30;
            int      maxTcpConnectionsPerEndpoint = 65535;

            Cosmos.PortReuseMode portReuseMode = Cosmos.PortReuseMode.PrivatePortPool;
            IWebProxy            webProxy      = new TestWebProxy();

            Cosmos.ConsistencyLevel consistencyLevel = Cosmos.ConsistencyLevel.ConsistentPrefix;

            CosmosClientBuilder cosmosClientBuilder = new CosmosClientBuilder(
                accountEndpoint: endpoint,
                authKeyOrResourceToken: key);

            CosmosClient        cosmosClient  = cosmosClientBuilder.Build(new MockDocumentClient());
            CosmosClientOptions clientOptions = cosmosClient.ClientOptions;

            Assert.AreEqual(endpoint, cosmosClient.Endpoint.OriginalString, "AccountEndpoint did not save correctly");
            Assert.AreEqual(key, cosmosClient.AccountKey, "AccountKey did not save correctly");

            //Verify the default values are different from the new values
            Assert.AreNotEqual(region, clientOptions.ApplicationRegion);
            Assert.IsNull(clientOptions.ApplicationPreferredRegions);
            Assert.AreNotEqual(connectionMode, clientOptions.ConnectionMode);
            Assert.AreNotEqual(maxConnections, clientOptions.GatewayModeMaxConnectionLimit);
            Assert.AreNotEqual(requestTimeout, clientOptions.RequestTimeout);
            Assert.AreNotEqual(userAgentSuffix, clientOptions.ApplicationName);
            Assert.AreNotEqual(apiType, clientOptions.ApiType);
            Assert.IsFalse(clientOptions.AllowBulkExecution);
            Assert.AreEqual(0, clientOptions.CustomHandlers.Count);
            Assert.IsNull(clientOptions.SerializerOptions);
            Assert.IsNotNull(clientOptions.Serializer);
            Assert.IsNull(clientOptions.WebProxy);
            Assert.IsFalse(clientOptions.LimitToEndpoint);
            Assert.IsTrue(clientOptions.EnableTcpConnectionEndpointRediscovery);
            Assert.IsNull(clientOptions.HttpClientFactory);
            Assert.AreNotEqual(consistencyLevel, clientOptions.ConsistencyLevel);
            Assert.IsFalse(clientOptions.EnablePartitionLevelFailover);

            //Verify GetConnectionPolicy returns the correct values for default
            ConnectionPolicy policy = clientOptions.GetConnectionPolicy(clientId: 0);

            Assert.AreEqual(ConnectionMode.Direct, policy.ConnectionMode);
            Assert.AreEqual(Protocol.Tcp, policy.ConnectionProtocol);
            Assert.AreEqual(clientOptions.GatewayModeMaxConnectionLimit, policy.MaxConnectionLimit);
            Assert.AreEqual(clientOptions.RequestTimeout, policy.RequestTimeout);
            Assert.IsNull(policy.IdleTcpConnectionTimeout);
            Assert.IsNull(policy.OpenTcpConnectionTimeout);
            Assert.IsNull(policy.MaxRequestsPerTcpConnection);
            Assert.IsNull(policy.MaxTcpConnectionsPerEndpoint);
            Assert.IsTrue(policy.EnableEndpointDiscovery);
            Assert.IsTrue(policy.EnableTcpConnectionEndpointRediscovery);
            Assert.IsNull(policy.HttpClientFactory);
            Assert.AreNotEqual(Cosmos.ConsistencyLevel.Session, clientOptions.ConsistencyLevel);
            Assert.IsFalse(policy.EnablePartitionLevelFailover);

            cosmosClientBuilder.WithApplicationRegion(region)
            .WithConnectionModeGateway(maxConnections, webProxy)
            .WithRequestTimeout(requestTimeout)
            .WithApplicationName(userAgentSuffix)
            .AddCustomHandlers(preProcessHandler)
            .WithApiType(apiType)
            .WithThrottlingRetryOptions(maxRetryWaitTime, maxRetryAttemptsOnThrottledRequests)
            .WithBulkExecution(true)
            .WithSerializerOptions(cosmosSerializerOptions)
            .WithConsistencyLevel(consistencyLevel)
            .WithPartitionLevelFailoverEnabled();

            cosmosClient  = cosmosClientBuilder.Build(new MockDocumentClient());
            clientOptions = cosmosClient.ClientOptions;

            //Verify all the values are updated
            Assert.AreEqual(region, clientOptions.ApplicationRegion);
            Assert.IsNull(clientOptions.ApplicationPreferredRegions);
            Assert.AreEqual(connectionMode, clientOptions.ConnectionMode);
            Assert.AreEqual(maxConnections, clientOptions.GatewayModeMaxConnectionLimit);
            Assert.AreEqual(requestTimeout, clientOptions.RequestTimeout);
            Assert.AreEqual(userAgentSuffix, clientOptions.ApplicationName);
            Assert.AreEqual(preProcessHandler, clientOptions.CustomHandlers[0]);
            Assert.AreEqual(apiType, clientOptions.ApiType);
            Assert.AreEqual(maxRetryAttemptsOnThrottledRequests, clientOptions.MaxRetryAttemptsOnRateLimitedRequests);
            Assert.AreEqual(maxRetryWaitTime, clientOptions.MaxRetryWaitTimeOnRateLimitedRequests);
            Assert.AreEqual(cosmosSerializerOptions.IgnoreNullValues, clientOptions.SerializerOptions.IgnoreNullValues);
            Assert.AreEqual(cosmosSerializerOptions.PropertyNamingPolicy, clientOptions.SerializerOptions.PropertyNamingPolicy);
            Assert.AreEqual(cosmosSerializerOptions.Indented, clientOptions.SerializerOptions.Indented);
            Assert.IsTrue(object.ReferenceEquals(webProxy, clientOptions.WebProxy));
            Assert.IsTrue(clientOptions.AllowBulkExecution);
            Assert.AreEqual(consistencyLevel, clientOptions.ConsistencyLevel);
            Assert.IsTrue(clientOptions.EnablePartitionLevelFailover);

            //Verify GetConnectionPolicy returns the correct values
            policy = clientOptions.GetConnectionPolicy(clientId: 0);
            Assert.AreEqual(region, policy.PreferredLocations[0]);
            Assert.AreEqual(ConnectionMode.Gateway, policy.ConnectionMode);
            Assert.AreEqual(Protocol.Https, policy.ConnectionProtocol);
            Assert.AreEqual(maxConnections, policy.MaxConnectionLimit);
            Assert.AreEqual(requestTimeout, policy.RequestTimeout);
            Assert.IsTrue(policy.UserAgentSuffix.Contains(userAgentSuffix));
            Assert.IsTrue(policy.UseMultipleWriteLocations);
            Assert.AreEqual(maxRetryAttemptsOnThrottledRequests, policy.RetryOptions.MaxRetryAttemptsOnThrottledRequests);
            Assert.AreEqual((int)maxRetryWaitTime.TotalSeconds, policy.RetryOptions.MaxRetryWaitTimeInSeconds);
            Assert.AreEqual((Documents.ConsistencyLevel)consistencyLevel, clientOptions.GetDocumentsConsistencyLevel());
            Assert.IsTrue(policy.EnablePartitionLevelFailover);

            IReadOnlyList <string> preferredLocations = new List <string>()
            {
                Regions.AustraliaCentral, Regions.AustraliaCentral2
            };

            //Verify Direct Mode settings
            cosmosClientBuilder = new CosmosClientBuilder(
                accountEndpoint: endpoint,
                authKeyOrResourceToken: key);
            cosmosClientBuilder.WithConnectionModeDirect(
                idleTcpConnectionTimeout,
                openTcpConnectionTimeout,
                maxRequestsPerTcpConnection,
                maxTcpConnectionsPerEndpoint,
                portReuseMode,
                enableTcpConnectionEndpointRediscovery)
            .WithApplicationPreferredRegions(preferredLocations);

            cosmosClient  = cosmosClientBuilder.Build(new MockDocumentClient());
            clientOptions = cosmosClient.ClientOptions;
            //Verify all the values are updated
            Assert.AreEqual(idleTcpConnectionTimeout, clientOptions.IdleTcpConnectionTimeout);
            Assert.AreEqual(openTcpConnectionTimeout, clientOptions.OpenTcpConnectionTimeout);
            Assert.AreEqual(maxRequestsPerTcpConnection, clientOptions.MaxRequestsPerTcpConnection);
            Assert.AreEqual(maxTcpConnectionsPerEndpoint, clientOptions.MaxTcpConnectionsPerEndpoint);
            Assert.AreEqual(portReuseMode, clientOptions.PortReuseMode);
            Assert.IsTrue(clientOptions.EnableTcpConnectionEndpointRediscovery);
            CollectionAssert.AreEqual(preferredLocations.ToArray(), clientOptions.ApplicationPreferredRegions.ToArray());

            //Verify GetConnectionPolicy returns the correct values
            policy = clientOptions.GetConnectionPolicy(clientId: 0);
            Assert.AreEqual(idleTcpConnectionTimeout, policy.IdleTcpConnectionTimeout);
            Assert.AreEqual(openTcpConnectionTimeout, policy.OpenTcpConnectionTimeout);
            Assert.AreEqual(maxRequestsPerTcpConnection, policy.MaxRequestsPerTcpConnection);
            Assert.AreEqual(maxTcpConnectionsPerEndpoint, policy.MaxTcpConnectionsPerEndpoint);
            Assert.AreEqual(portReuseMode, policy.PortReuseMode);
            Assert.IsTrue(policy.EnableTcpConnectionEndpointRediscovery);
            CollectionAssert.AreEqual(preferredLocations.ToArray(), policy.PreferredLocations.ToArray());
        }
示例#12
0
 public RunProcessor(IFailureAnalyzer failureAnalyzer, ILogger <RunProcessor> logger, IMemoryCache cache, SecretClient secretClient, CosmosClient cosmosClient, VssConnection vssConnection)
 {
     this.failureAnalyzer = failureAnalyzer;
     this.logger          = logger;
     this.cache           = cache;
     this.secretClient    = secretClient;
     this.cosmosClient    = cosmosClient;
     this.vssConnection   = vssConnection;
 }
示例#13
0
        public async Task ReadMany404ExceptionTest()
        {
            Database database = await this.cosmosClient.CreateDatabaseAsync(Guid.NewGuid().ToString());

            Container container = await database.CreateContainerAsync(Guid.NewGuid().ToString(), "/pk");

            for (int i = 0; i < 5; i++)
            {
                await container.CreateItemAsync(
                    ToDoActivity.CreateRandomToDoActivity("pk" + i, i.ToString()));
            }

            List <(string, PartitionKey)> itemList = new List <(string, PartitionKey)>();

            for (int i = 0; i < 5; i++)
            {
                itemList.Add((i.ToString(), new PartitionKey("pk" + i)));
            }

            // 429 test
            //List<Task> tasks = new List<Task>();
            //ConcurrentQueue<ResponseMessage> failedRequests = new ConcurrentQueue<ResponseMessage>();
            //for (int i = 0; i < 500; i++)
            //{
            //    tasks.Add(Task.Run(async () =>
            //    {
            //        ResponseMessage responseMessage = await container.ReadManyItemsStreamAsync(itemList);
            //        if (!responseMessage.IsSuccessStatusCode)
            //        {
            //            failedRequests.Enqueue(responseMessage);
            //            Assert.AreEqual(responseMessage.StatusCode, HttpStatusCode.TooManyRequests);
            //        }
            //    }));
            //}

            //await Task.WhenAll(tasks);
            //Assert.IsTrue(failedRequests.Count > 0);

            await container.ReadManyItemsAsync <ToDoActivity>(itemList); // Warm up caches

            using (CosmosClient cosmosClient = TestCommon.CreateCosmosClient())
            {
                Container newContainer = cosmosClient.GetContainer(database.Id, container.Id);
                await newContainer.DeleteContainerAsync();
            }

            using (ResponseMessage responseMessage = await container.ReadManyItemsStreamAsync(itemList))
            {
                Assert.AreEqual(responseMessage.StatusCode, HttpStatusCode.NotFound);
            }

            try
            {
                await container.ReadManyItemsAsync <ToDoActivity>(itemList);

                Assert.Fail("Typed API should throw");
            }
            catch (CosmosException ex)
            {
                Assert.AreEqual(ex.Error.Code, "NotFound");
            }

            await database.DeleteAsync();
        }
示例#14
0
        private async Task StartDemo()
        {
            Console.WriteLine("Starting Cosmos DB SQL API Demo!");

            //Create a new demo database
            string databaseName = "demoDB_" + Guid.NewGuid().ToString().Substring(0, 5);

            this.SendMessageToConsoleAndWait($"Creating database {databaseName}...");

            this.client   = new CosmosClient(EndpointUri, Key);
            this.database = await this.client.CreateDatabaseIfNotExistsAsync(databaseName);

            //Create a new demo collection inside the demo database.
            //This creates a collection with a reserved throughput. You can customize the options using a ContainerProperties object
            //This operation has pricing implications.
            string containerName = "collection_" + Guid.NewGuid().ToString().Substring(0, 5);


            this.SendMessageToConsoleAndWait($"Creating collection demo{containerName}...");

            this.container = await this.database.CreateContainerIfNotExistsAsync(containerName, "/LastName");

            //Create some documents in the collection
            Person person1 = new Person
            {
                Id        = "Person.1",
                FirstName = "Santiago",
                LastName  = "Fernandez",
                Devices   = new Device[]
                {
                    new Device {
                        OperatingSystem = "iOS", CameraMegaPixels = 7,
                        Ram             = 16, Usage = "Personal"
                    },
                    new Device {
                        OperatingSystem = "Android", CameraMegaPixels = 12,
                        Ram             = 64, Usage = "Work"
                    }
                },
                Gender  = "Male",
                Address = new Address

                {
                    City       = "Seville",
                    Country    = "Spain",
                    PostalCode = "28973",
                    Street     = "Diagonal",
                    State      = "Andalucia"
                },
                IsRegistered = true
            };


            await this.CreateDocumentIfNotExistsAsync(databaseName, containerName, person1);

            Person person2 = new Person
            {
                Id        = "Person.2",
                FirstName = "Agatha",
                LastName  = "Smith",
                Devices   = new Device[]
                {
                    new Device {
                        OperatingSystem = "iOS", CameraMegaPixels = 12,
                        Ram             = 32, Usage = "Work"
                    },
                    new Device {
                        OperatingSystem = "Windows", CameraMegaPixels = 12,
                        Ram             = 64, Usage = "Personal"
                    }
                },
                Gender  = "Female",
                Address = new Address
                {
                    City       = "Laguna Beach",
                    Country    = "United States",
                    PostalCode = "12345",
                    Street     = "Main",
                    State      = "CA"
                },
                IsRegistered = true
            };


            await this.CreateDocumentIfNotExistsAsync(databaseName, containerName, person2);

            //Make some queries to the collection
            this.SendMessageToConsoleAndWait($"Getting documents from the collection{containerName}...");

            //Find documents using LINQ

            IQueryable <Person> queryablePeople = this.container.GetItemLinqQueryable <Person>(true)
                                                  .Where(p => p.Gender == "Male");

            System.Console.WriteLine("Running LINQ query for finding men...");
            foreach (Person foundPerson in queryablePeople)
            {
                System.Console.WriteLine($"\tPerson: {foundPerson}");
            }

            //Find documents using SQL

            var sqlQuery = "SELECT * FROM Person WHERE Person.Gender = 'Female'";

            QueryDefinition       queryDefinition         = new QueryDefinition(sqlQuery);
            FeedIterator <Person> peopleResultSetIterator = this.container.GetItemQueryIterator <Person>(queryDefinition);

            System.Console.WriteLine("Running SQL query for finding women...");
            while (peopleResultSetIterator.HasMoreResults)
            {
                FeedResponse <Person> currentResultSet = await peopleResultSetIterator.ReadNextAsync();

                foreach (Person foundPerson in currentResultSet)
                {
                    System.Console.WriteLine($"\tPerson: {foundPerson}");
                }
            }

            Console.WriteLine("Press any key to continue...");

            Console.ReadKey();

            //Update documents in a collection
            this.SendMessageToConsoleAndWait($"Updating documents in the collection { containerName}...");
            person2.FirstName = "Mathew";
            person2.Gender    = "Male";

            await this.container.UpsertItemAsync(person2);

            this.SendMessageToConsoleAndWait($"Document modified {person2}");

            //Delete a single document from the collection
            this.SendMessageToConsoleAndWait($"Deleting documents from the collection { containerName}...");

            PartitionKey partitionKey = new PartitionKey(person1.LastName);

            await this.container.DeleteItemAsync <Person>(person1.Id, partitionKey);

            this.SendMessageToConsoleAndWait($"Document deleted {person1}");

            //Delete created demo database and all its children elements
            this.SendMessageToConsoleAndWait("Cleaning-up your Cosmos DB account...");
            await this.database.DeleteAsync();
        }
        public async Task ShouldReturnNotModifiedAfterCyclingOnAllRanges()
        {
            // Setting mock to have 3 ranges, this test will get a 304 on all 3 ranges, do 3 backend requests, and return a 304
            MultiRangeMockDocumentClient documentClient = new MultiRangeMockDocumentClient();
            CosmosClient client = MockCosmosUtil.CreateMockCosmosClient();
            Mock <CosmosClientContext> mockContext = new Mock <CosmosClientContext>();

            mockContext.Setup(x => x.ClientOptions).Returns(MockCosmosUtil.GetDefaultConfiguration());
            mockContext.Setup(x => x.DocumentClient).Returns(documentClient);
            mockContext.Setup(x => x.SerializerCore).Returns(MockCosmosUtil.Serializer);
            mockContext.Setup(x => x.Client).Returns(client);
            mockContext.Setup(x => x.CreateLink(It.IsAny <string>(), It.IsAny <string>(), It.IsAny <string>())).Returns(new Uri("/dbs/test/colls/test", UriKind.Relative));

            ResponseMessage firstResponse = new ResponseMessage(HttpStatusCode.NotModified);

            firstResponse.Headers.ETag = "FirstContinuation";
            ResponseMessage secondResponse = new ResponseMessage(HttpStatusCode.NotModified);

            secondResponse.Headers.ETag = "SecondContinuation";
            ResponseMessage thirdResponse = new ResponseMessage(HttpStatusCode.NotModified);

            thirdResponse.Headers.ETag = "ThirdContinuation";

            mockContext.SetupSequence(x => x.ProcessResourceOperationAsync <ResponseMessage>(
                                          It.IsAny <Uri>(),
                                          It.IsAny <Documents.ResourceType>(),
                                          It.IsAny <Documents.OperationType>(),
                                          It.IsAny <RequestOptions>(),
                                          It.IsAny <ContainerCore>(),
                                          It.IsAny <PartitionKey?>(),
                                          It.IsAny <Stream>(),
                                          It.IsAny <Action <RequestMessage> >(),
                                          It.IsAny <Func <ResponseMessage, ResponseMessage> >(),
                                          It.IsAny <CosmosDiagnosticsContext>(),
                                          It.IsAny <CancellationToken>()))
            .Returns(Task.FromResult(firstResponse))
            .Returns(Task.FromResult(secondResponse))
            .Returns(Task.FromResult(thirdResponse));

            DatabaseCore databaseCore = new DatabaseCore(mockContext.Object, "mydb");

            StandByFeedIteratorCore iterator = new StandByFeedIteratorCore(
                mockContext.Object, new ContainerCore(mockContext.Object, databaseCore, "myColl"), null, 10, new ChangeFeedRequestOptions());
            ResponseMessage firstRequest = await iterator.ReadNextAsync();

            Assert.IsTrue(firstRequest.Headers.ContinuationToken.Contains(firstResponse.Headers.ETag), "Response should contain the first continuation");
            Assert.IsTrue(firstRequest.Headers.ContinuationToken.Contains(secondResponse.Headers.ETag), "Response should contain the second continuation");
            Assert.IsTrue(firstRequest.Headers.ContinuationToken.Contains(thirdResponse.Headers.ETag), "Response should contain the third continuation");
            Assert.AreEqual(HttpStatusCode.NotModified, firstRequest.StatusCode);

            mockContext.Verify(x => x.ProcessResourceOperationAsync <ResponseMessage>(
                                   It.IsAny <Uri>(),
                                   It.IsAny <Documents.ResourceType>(),
                                   It.IsAny <Documents.OperationType>(),
                                   It.IsAny <RequestOptions>(),
                                   It.IsAny <ContainerCore>(),
                                   It.IsAny <PartitionKey?>(),
                                   It.IsAny <Stream>(),
                                   It.IsAny <Action <RequestMessage> >(),
                                   It.IsAny <Func <ResponseMessage, ResponseMessage> >(),
                                   It.IsAny <CosmosDiagnosticsContext>(),
                                   It.IsAny <CancellationToken>()), Times.Exactly(3));
        }
示例#16
0
        static async Task Main(string[] args)
        {
            string DatabaseName  = "bulk-tutorial";
            string ContainerName = "items";
            string Account;
            string Key;
            int    ItemsToInsert = 300000;

            if (ParseCommandLine(args, out Account, out Key, out DatabaseName, out ContainerName, out ItemsToInsert))
            {
                string       EndpointUrl      = $"https://{Account}.documents.azure.com:443/";
                string       AuthorizationKey = Key;
                CosmosClient cosmosClient     = new CosmosClient(EndpointUrl, AuthorizationKey, new CosmosClientOptions()
                {
                    AllowBulkExecution = true
                });



                // Create with a throughput of 50000 RU/s
                // Indexing Policy to exclude all attributes to maximize RU/s usage
                Console.WriteLine("This sample app will create a 50000 RU/s container, press any key to continue.");
                Console.ReadKey();



                // <Initialize>
                Database database = await cosmosClient.CreateDatabaseIfNotExistsAsync(DatabaseName);

                await database.DefineContainer(ContainerName, "/pk")
                .WithIndexingPolicy()
                .WithIndexingMode(IndexingMode.Consistent)
                .WithIncludedPaths()
                .Attach()
                .WithExcludedPaths()
                .Path("/*")
                .Attach()
                .Attach()
                .CreateAsync(50000);

                // </Initialize>



                try
                {
                    // Prepare items for insertion
                    Console.WriteLine($"Preparing {ItemsToInsert} items to insert...");
                    // <Operations>

                    Dictionary <PartitionKey, Stream> itemsToInsert = new Dictionary <PartitionKey, Stream>(ItemsToInsert);
                    foreach (Item item in Program.GetItemsToInsert(ItemsToInsert))
                    {
                        MemoryStream stream = new MemoryStream();
                        await JsonSerializer.SerializeAsync(stream, item);

                        itemsToInsert.Add(new PartitionKey(item.pk), stream);
                    }

                    // </Operations>



                    // Create the list of Tasks
                    Console.WriteLine($"Starting...");
                    Stopwatch stopwatch = Stopwatch.StartNew();

                    // <ConcurrentTasks>
                    Container   container = database.GetContainer(ContainerName);
                    List <Task> tasks     = new List <Task>(ItemsToInsert);

                    foreach (KeyValuePair <PartitionKey, Stream> item in itemsToInsert)
                    {
                        tasks.Add(container.CreateItemStreamAsync(item.Value, item.Key)
                                  .ContinueWith((Task <ResponseMessage> task) =>
                        {
                            using (ResponseMessage response = task.Result)
                            {
                                if (!response.IsSuccessStatusCode)
                                {
                                    Console.WriteLine($"Received {response.StatusCode} ({response.ErrorMessage}) status code for operation {EndpointUrl}.");
                                }
                            }
                        }));
                    }



                    // Wait until all are done
                    await Task.WhenAll(tasks);

                    // </ConcurrentTasks>

                    stopwatch.Stop();

                    Console.WriteLine($"Finished in writing {ItemsToInsert} items in {stopwatch.Elapsed}.");
                }

                catch (Exception ex)
                {
                    Console.WriteLine(ex);
                }

                finally
                {
                    Console.WriteLine("This sample app will delete the database, press any key to continue.");
                    Console.ReadKey();
                    Console.WriteLine("Cleaning up resources...");
                    await database.DeleteAsync();
                }
            }
        }
        public async Task ContinuationTokenIsNotUpdatedOnFails()
        {
            MultiRangeMockDocumentClient documentClient = new MultiRangeMockDocumentClient();
            CosmosClient client = MockCosmosUtil.CreateMockCosmosClient();
            Mock <CosmosClientContext> mockContext = new Mock <CosmosClientContext>();

            mockContext.Setup(x => x.ClientOptions).Returns(MockCosmosUtil.GetDefaultConfiguration());
            mockContext.Setup(x => x.DocumentClient).Returns(documentClient);
            mockContext.Setup(x => x.SerializerCore).Returns(MockCosmosUtil.Serializer);
            mockContext.Setup(x => x.Client).Returns(client);
            mockContext.Setup(x => x.CreateLink(It.IsAny <string>(), It.IsAny <string>(), It.IsAny <string>())).Returns(new Uri("/dbs/test/colls/test", UriKind.Relative));

            ResponseMessage firstResponse = new ResponseMessage(HttpStatusCode.NotModified);

            firstResponse.Headers.ETag = "FirstContinuation";
            ResponseMessage secondResponse = new ResponseMessage(
                statusCode: HttpStatusCode.NotFound,
                requestMessage: null,
                headers: new Headers()
            {
                ETag = "ShouldNotContainThis"
            },
                cosmosException: CosmosExceptionFactory.CreateNotFoundException("something"),
                diagnostics: new CosmosDiagnosticsContextCore());

            mockContext.SetupSequence(x => x.ProcessResourceOperationAsync <ResponseMessage>(
                                          It.IsAny <Uri>(),
                                          It.IsAny <Documents.ResourceType>(),
                                          It.IsAny <Documents.OperationType>(),
                                          It.IsAny <RequestOptions>(),
                                          It.IsAny <ContainerCore>(),
                                          It.IsAny <PartitionKey?>(),
                                          It.IsAny <Stream>(),
                                          It.IsAny <Action <RequestMessage> >(),
                                          It.IsAny <Func <ResponseMessage, ResponseMessage> >(),
                                          It.IsAny <CosmosDiagnosticsContext>(),
                                          It.IsAny <CancellationToken>()))
            .Returns(Task.FromResult(firstResponse))
            .Returns(Task.FromResult(secondResponse));

            DatabaseCore databaseCore = new DatabaseCore(mockContext.Object, "mydb");

            StandByFeedIteratorCore iterator = new StandByFeedIteratorCore(
                mockContext.Object, new ContainerCore(mockContext.Object, databaseCore, "myColl"), null, 10, new ChangeFeedRequestOptions());
            ResponseMessage firstRequest = await iterator.ReadNextAsync();

            Assert.IsTrue(firstRequest.Headers.ContinuationToken.Contains(firstResponse.Headers.ETag), "Response should contain the first continuation");
            Assert.IsTrue(!firstRequest.Headers.ContinuationToken.Contains(secondResponse.Headers.ETag), "Response should not contain the second continuation");
            Assert.AreEqual(HttpStatusCode.NotFound, firstRequest.StatusCode);

            mockContext.Verify(x => x.ProcessResourceOperationAsync <ResponseMessage>(
                                   It.IsAny <Uri>(),
                                   It.IsAny <Documents.ResourceType>(),
                                   It.IsAny <Documents.OperationType>(),
                                   It.IsAny <RequestOptions>(),
                                   It.IsAny <ContainerCore>(),
                                   It.IsAny <PartitionKey?>(),
                                   It.IsAny <Stream>(),
                                   It.IsAny <Action <RequestMessage> >(),
                                   It.IsAny <Func <ResponseMessage, ResponseMessage> >(),
                                   It.IsAny <CosmosDiagnosticsContext>(),
                                   It.IsAny <CancellationToken>()), Times.Exactly(2));
        }
 // Use Dependency Injection to inject the HttpClientFactory service and Cosmos DB client that were configured in Startup.cs.
 public Functions(IHttpClientFactory httpClientFactory, CosmosClient cosmosClient)
 {
     _httpClientFactory = httpClientFactory;
     _cosmosClient      = cosmosClient;
 }