// </RunSimpleScript>
/// <summary>
/// Import many documents using stored procedure.
/// </summary>
// <RunBulkImport>
private static async Task RunBulkImport(Container container)
{
string inputDirectory = @".\Data\";
string inputFileMask = "*.json";
int maxFiles = 2000;
int maxScriptSize = 50000;
// 1. Get the files.
string[] fileNames = Directory.GetFiles(inputDirectory, inputFileMask);
DirectoryInfo di = new DirectoryInfo(inputDirectory);
FileInfo[] fileInfos = di.GetFiles(inputFileMask);
// 2. Prepare for import.
int currentCount = 0;
int fileCount = maxFiles != 0 ? Math.Min(maxFiles, fileNames.Length) : fileNames.Length;
// 3. Create stored procedure for this script.
string scriptId = "BulkImport";
string body = File.ReadAllText(@".\JS\BulkImport.js");
await TryDeleteStoredProcedure(container, scriptId);
Scripts cosmosScripts = container.Scripts;
StoredProcedureResponse sproc = await cosmosScripts.CreateStoredProcedureAsync(new StoredProcedureProperties(scriptId, body));
// 4. Create a batch of docs (MAX is limited by request size (2M) and to script for execution.
// We send batches of documents to create to script.
// Each batch size is determined by MaxScriptSize.
// MaxScriptSize should be so that:
// -- it fits into one request (MAX request size is 16Kb).
// -- it doesn't cause the script to time out.
// -- it is possible to experiment with MaxScriptSize to get best performance given number of throttles, etc.
while (currentCount < fileCount)
{
// 5. Create args for current batch.
// Note that we could send a string with serialized JSON and JSON.parse it on the script side,
// but that would cause script to run longer. Since script has timeout, unload the script as much
// as we can and do the parsing by client and framework. The script will get JavaScript objects.
string argsJson = CreateBulkInsertScriptArguments(fileNames, currentCount, fileCount, maxScriptSize);
dynamic[] args = new dynamic[] { JsonConvert.DeserializeObject <dynamic>(argsJson) };
// 6. execute the batch.
StoredProcedureExecuteResponse <int> scriptResult = await cosmosScripts.ExecuteStoredProcedureAsync <int>(
scriptId,
new PartitionKey("Andersen"),
args);
// 7. Prepare for next batch.
int currentlyInserted = scriptResult.Resource;
currentCount += currentlyInserted;
}
// 8. Validate
int numDocs = 0;
FeedIterator <dynamic> setIterator = container.GetItemQueryIterator <dynamic>();
while (setIterator.HasMoreResults)
{
FeedResponse <dynamic> response = await setIterator.ReadNextAsync();
numDocs += response.Count();
}
Console.WriteLine("Found {0} documents in the collection. There were originally {1} files in the Data directory\r\n", numDocs, fileCount);
}
// Requests clustering data from the database
public static async Task <List <ClusterPersonalDetailsWithoutNull> > QueryClusterDetailsAsync()
{
var sqlQueryText = "SELECT c.document.Extraversion, c.document.Agreeableness, c.document.Conscientiousness, c.document.Neuroticism, c.document.Openness FROM c";
QueryDefinition queryDefinition = new QueryDefinition(sqlQueryText);
CosmosClient cosmosClient = new CosmosClient(cosmosServiceEndpoint, cosmosDBKey);
Azure.Cosmos.Database database;
database = await cosmosClient.CreateDatabaseIfNotExistsAsync(cosmosDBDatabaseName);
Container container;
container = await database.CreateContainerIfNotExistsAsync(cosmosDBConteinerId, "/id");
FeedIterator <ClusterPersonalDetails> queryResultSetIterator = container.GetItemQueryIterator <ClusterPersonalDetails>(queryDefinition);
List <ClusterPersonalDetails> detailsList = new List <ClusterPersonalDetails>();
while (queryResultSetIterator.HasMoreResults)
{
Azure.Cosmos.FeedResponse <ClusterPersonalDetails> currentResultSet = await queryResultSetIterator.ReadNextAsync();
foreach (ClusterPersonalDetails details in currentResultSet)
{
detailsList.Add(details);
}
}
return(RemoveNullValues(detailsList));
}
public async Task Schema_OnV2MigrationMaintainPartitionId()
{
IEnumerable <int> expectedIds = Enumerable.Range(0, 20);
List <int> receivedIds = new List <int>();
ChangeFeedProcessor processor = this.Container
.GetChangeFeedProcessorBuilder("test", (IReadOnlyCollection <TestClass> docs, CancellationToken token) =>
{
foreach (TestClass doc in docs)
{
receivedIds.Add(int.Parse(doc.id));
}
return(Task.CompletedTask);
})
.WithInstanceName("random")
.WithLeaseContainer(this.LeaseContainer).Build();
await processor.StartAsync();
// Letting processor initialize
await Task.Delay(BaseChangeFeedClientHelper.ChangeFeedSetupTime);
// Inserting some documents
foreach (int id in expectedIds.Take(10))
{
await this.Container.CreateItemAsync <dynamic>(new { id = id.ToString() });
}
// Waiting on all notifications to finish
await Task.Delay(BaseChangeFeedClientHelper.ChangeFeedCleanupTime);
await processor.StopAsync();
// At this point we have leases for V3, so we will simulate V2 by manually adding PartitionId and removing LeaseToken
using FeedIterator <JObject> iterator = this.LeaseContainer.GetItemQueryIterator <JObject>();
while (iterator.HasMoreResults)
{
FeedResponse <JObject> page = await iterator.ReadNextAsync();
foreach (JObject lease in page)
{
string leaseId = lease.Value <string>("id");
if (leaseId.Contains(".info") || leaseId.Contains(".lock"))
{
// These are the store initialization marks
continue;
}
// create the PartitionId property
lease.Add("PartitionId", lease.Value <string>("LeaseToken"));
lease.Remove("LeaseToken");
await this.LeaseContainer.UpsertItemAsync <JObject>(lease);
}
}
// Now all leases are V2 leases, create the rest of the documents
foreach (int id in expectedIds.TakeLast(10))
{
await this.Container.CreateItemAsync <dynamic>(new { id = id.ToString() });
}
await processor.StartAsync();
// Letting processor initialize
await Task.Delay(BaseChangeFeedClientHelper.ChangeFeedSetupTime);
// Waiting on all notifications to finish, should be using PartitionId from the V2 lease
await Task.Delay(BaseChangeFeedClientHelper.ChangeFeedCleanupTime);
await processor.StopAsync();
// Verify we processed all items (including when using the V2 leases)
CollectionAssert.AreEqual(expectedIds.ToList(), receivedIds);
// Verify the after-migration leases have both PartitionId and LeaseToken with the same value
using FeedIterator <dynamic> iteratorAfter = this.LeaseContainer.GetItemQueryIterator <dynamic>();
while (iteratorAfter.HasMoreResults)
{
FeedResponse <dynamic> page = await iteratorAfter.ReadNextAsync();
foreach (dynamic lease in page)
{
string leaseId = lease.id;
if (leaseId.Contains(".info") || leaseId.Contains(".lock"))
{
// These are the store initialization marks
continue;
}
Assert.IsNotNull(lease.LeaseToken, "LeaseToken is missing after migration of lease schema");
Assert.IsNotNull(lease.PartitionId, "PartitionId is missing after migration of lease schema");
Assert.AreEqual(lease.LeaseToken, lease.PartitionId, "LeaseToken and PartitionId should be equal after migration");
}
}
}
public async Task ChangeFeedIteratorCore_WithFullFidelity()
{
ContainerProperties properties = new ContainerProperties(id: Guid.NewGuid().ToString(), partitionKeyPath: ChangeFeedIteratorCoreTests.PartitionKey);
properties.ChangeFeedPolicy.FullFidelityRetention = TimeSpan.FromMinutes(5);
ContainerResponse response = await this.database.CreateContainerAsync(
properties,
cancellationToken : this.cancellationToken);
ContainerInternal container = (ContainerInternal)response;
// FF does not work with StartFromBeginning currently, so we capture an initial continuation.
FeedIterator <ToDoActivityWithMetadata> fullFidelityIterator = container.GetChangeFeedIterator <ToDoActivityWithMetadata>(
ChangeFeedStartFrom.Now(),
ChangeFeedMode.FullFidelity);
string initialContinuation = null;
while (fullFidelityIterator.HasMoreResults)
{
try
{
FeedResponse <ToDoActivityWithMetadata> feedResponse = await fullFidelityIterator.ReadNextAsync(this.cancellationToken);
initialContinuation = feedResponse.ContinuationToken;
}
catch (CosmosException cosmosException) when(cosmosException.StatusCode == HttpStatusCode.NotModified)
{
initialContinuation = cosmosException.Headers.ContinuationToken;
break;
}
}
// Insert documents and then delete them
int totalDocuments = 50;
IList <ToDoActivity> createdItems = await this.CreateRandomItems(container, totalDocuments, randomPartitionKey : true);
foreach (ToDoActivity item in createdItems)
{
await container.DeleteItemAsync <ToDoActivity>(item.id, new PartitionKey(item.pk));
}
// Resume Change Feed and verify we pickup all the events
fullFidelityIterator = container.GetChangeFeedIterator <ToDoActivityWithMetadata>(
ChangeFeedStartFrom.ContinuationToken(initialContinuation),
ChangeFeedMode.FullFidelity);
int detectedEvents = 0;
bool hasInserts = false;
bool hasDeletes = false;
while (fullFidelityIterator.HasMoreResults)
{
try
{
FeedResponse <ToDoActivityWithMetadata> feedResponse = await fullFidelityIterator.ReadNextAsync(this.cancellationToken);
foreach (ToDoActivityWithMetadata item in feedResponse)
{
Assert.IsNotNull(item.metadata, "Metadata not present");
Assert.IsNotNull(item.metadata.operationType, "Metadata has no operationType");
hasInserts |= item.metadata.operationType == "create";
hasDeletes |= item.metadata.operationType == "delete";
}
detectedEvents += feedResponse.Count;
}
catch (CosmosException cosmosException) when(cosmosException.StatusCode == HttpStatusCode.NotModified)
{
break;
}
}
Assert.AreEqual(2 * totalDocuments, detectedEvents, "Full Fidelity should include inserts and delete events.");
Assert.IsTrue(hasInserts, "No metadata for create operationType found");
Assert.IsTrue(hasDeletes, "No metadata for delete operationType found");
}
// </ReadItemAsync>
// <QueryItems>
private static async Task QueryItems()
{
//******************************************************************************************************************
// 1.4 - Query for items by a property other than Id
//
// NOTE: Operations like AsEnumerable(), ToList(), ToArray() will make as many trips to the database
// as required to fetch the entire result-set. Even if you set MaxItemCount to a smaller number.
// MaxItemCount just controls how many results to fetch each trip.
//******************************************************************************************************************
Console.WriteLine("\n1.4 - Querying for a item using its AccountNumber property");
QueryDefinition query = new QueryDefinition(
"select * from sales s where s.AccountNumber = @AccountInput ")
.WithParameter("@AccountInput", "Account1");
FeedIterator <SalesOrder> resultSet = container.GetItemQueryIterator <SalesOrder>(
query,
requestOptions: new QueryRequestOptions()
{
PartitionKey = new PartitionKey("Account1"),
MaxItemCount = 1
});
List <SalesOrder> allSalesForAccount1 = new List <SalesOrder>();
while (resultSet.HasMoreResults)
{
SalesOrder sale = (await resultSet.ReadNextAsync()).First();
Console.WriteLine($"\n1.4.1 Account Number: {sale.AccountNumber}; Id: {sale.Id} ");
allSalesForAccount1.Add(sale);
}
Console.WriteLine($"\n1.4.2 Query found {allSalesForAccount1.Count} items.");
// Use the same query as before but get the cosmos response message to access the stream directly
FeedIterator streamResultSet = container.GetItemQueryStreamIterator(
query,
requestOptions: new QueryRequestOptions()
{
PartitionKey = new PartitionKey("Account1"),
MaxItemCount = 10,
MaxConcurrency = 1
});
List <SalesOrder> allSalesForAccount1FromStream = new List <SalesOrder>();
while (streamResultSet.HasMoreResults)
{
using (ResponseMessage responseMessage = await streamResultSet.ReadNextAsync())
{
// Item stream operations do not throw exceptions for better performance
if (responseMessage.IsSuccessStatusCode)
{
dynamic streamResponse = FromStream <dynamic>(responseMessage.Content);
List <SalesOrder> salesOrders = streamResponse.Documents.ToObject <List <SalesOrder> >();
Console.WriteLine($"\n1.4.3 - Item Query via stream {salesOrders.Count}");
allSalesForAccount1FromStream.AddRange(salesOrders);
}
else
{
Console.WriteLine($"Query item from stream failed. Status code: {responseMessage.StatusCode} Message: {responseMessage.ErrorMessage}");
}
}
}
Console.WriteLine($"\n1.4.4 Query found {allSalesForAccount1FromStream.Count} items.");
if (allSalesForAccount1.Count != allSalesForAccount1FromStream.Count)
{
throw new InvalidDataException($"Both query operations should return the same list");
}
}
public async Task TestQueryWithPartitionKeyAsync()
{
string[] inputDocs = new[]
{
@"{""id"":""documentId1"",""key"":""A"",""prop"":3,""shortArray"":[{""a"":5}]}",
@"{""id"":""documentId2"",""key"":""A"",""prop"":2,""shortArray"":[{""a"":6}]}",
@"{""id"":""documentId3"",""key"":""A"",""prop"":1,""shortArray"":[{""a"":7}]}",
@"{""id"":""documentId4"",""key"":5,""prop"":3,""shortArray"":[{""a"":5}]}",
@"{""id"":""documentId5"",""key"":5,""prop"":2,""shortArray"":[{""a"":6}]}",
@"{""id"":""documentId6"",""key"":5,""prop"":1,""shortArray"":[{""a"":7}]}",
@"{""id"":""documentId10"",""prop"":3,""shortArray"":[{""a"":5}]}",
@"{""id"":""documentId11"",""prop"":2,""shortArray"":[{""a"":6}]}",
@"{""id"":""documentId12"",""prop"":1,""shortArray"":[{""a"":7}]}",
};
await this.CreateIngestQueryDeleteAsync(
ConnectionModes.Direct | ConnectionModes.Gateway,
CollectionTypes.SinglePartition | CollectionTypes.MultiPartition,
inputDocs,
ImplementationAsync,
"/key");
async Task ImplementationAsync(Container container, IReadOnlyList <CosmosObject> documents)
{
Assert.AreEqual(0, (await QueryTestsBase.RunQueryAsync(
container,
@"SELECT * FROM Root r WHERE false",
new QueryRequestOptions()
{
MaxConcurrency = 1,
})).Count);
object[] keys = new object[] { "A", 5, Undefined.Value };
for (int i = 0; i < keys.Length; ++i)
{
List <string> expected = documents
.Skip(i * 3)
.Take(3)
.Select(doc => ((CosmosString)doc["id"]).Value)
.ToList();
string expectedResult = string.Join(",", expected);
// Order-by
List <string> expectedCopy = new List <string>(expected);
expectedCopy.Reverse();
string expectedOrderByResult = string.Join(",", expectedCopy);
List <(string, string)> queries = new List <(string, string)>()
{
($@"SELECT * FROM Root r WHERE r.id IN (""{expected[0]}"", ""{expected[1]}"", ""{expected[2]}"")", expectedResult),
(@"SELECT * FROM Root r WHERE r.prop BETWEEN 1 AND 3", expectedResult),
(@"SELECT VALUE r FROM Root r JOIN c IN r.shortArray WHERE c.a BETWEEN 5 and 7", expectedResult),
($@"SELECT TOP 10 * FROM Root r WHERE r.id IN (""{expected[0]}"", ""{expected[1]}"", ""{expected[2]}"")", expectedResult),
(@"SELECT TOP 10 * FROM Root r WHERE r.prop BETWEEN 1 AND 3", expectedResult),
(@"SELECT TOP 10 VALUE r FROM Root r JOIN c IN r.shortArray WHERE c.a BETWEEN 5 and 7", expectedResult),
($@"SELECT * FROM Root r WHERE r.id IN (""{expected[0]}"", ""{expected[1]}"", ""{expected[2]}"") ORDER BY r.prop", expectedOrderByResult),
(@"SELECT * FROM Root r WHERE r.prop BETWEEN 1 AND 3 ORDER BY r.prop", expectedOrderByResult),
(@"SELECT VALUE r FROM Root r JOIN c IN r.shortArray WHERE c.a BETWEEN 5 and 7 ORDER BY r.prop", expectedOrderByResult),
};
if (i < keys.Length - 1)
{
string key;
if (keys[i] is string)
{
key = "'" + keys[i].ToString() + "'";
}
else
{
key = keys[i].ToString();
}
queries.Add((string.Format(CultureInfo.InvariantCulture, @"SELECT * FROM Root r WHERE r.key = {0} ORDER BY r.prop", key), expectedOrderByResult));
}
foreach ((string, string)queryAndExpectedResult in queries)
{
FeedIterator <Document> resultSetIterator = container.GetItemQueryIterator <Document>(
queryText: queryAndExpectedResult.Item1,
requestOptions: new QueryRequestOptions()
{
MaxItemCount = 1,
PartitionKey = new Cosmos.PartitionKey(keys[i]),
});
List <Document> result = new List <Document>();
while (resultSetIterator.HasMoreResults)
{
result.AddRange(await resultSetIterator.ReadNextAsync());
}
string resultDocIds = string.Join(",", result.Select(doc => doc.Id));
Assert.AreEqual(queryAndExpectedResult.Item2, resultDocIds);
}
}
}
}
public async Task StandByFeedIterator()
{
int totalCount = 0;
string lastcontinuation = string.Empty;
int firstRunTotal = 25;
int batchSize = 25;
Documents.Routing.Range <string> previousRange = null;
Documents.Routing.Range <string> currentRange = null;
int pkRangesCount = (await this.Container.ClientContext.DocumentClient.ReadPartitionKeyRangeFeedAsync(this.Container.LinkUri)).Count;
int visitedPkRanges = 0;
await this.CreateRandomItems(batchSize, randomPartitionKey : true);
ContainerCore itemsCore = (ContainerCore)this.Container;
FeedIterator feedIterator = itemsCore.GetStandByFeedIterator(requestOptions: new ChangeFeedRequestOptions()
{
StartTime = DateTime.MinValue
});
while (feedIterator.HasMoreResults)
{
using (ResponseMessage responseMessage =
await feedIterator.ReadNextAsync(this.cancellationToken))
{
lastcontinuation = responseMessage.Headers.ContinuationToken;
Assert.AreEqual(responseMessage.ContinuationToken, responseMessage.Headers.ContinuationToken);
List <CompositeContinuationToken> deserializedToken = JsonConvert.DeserializeObject <List <CompositeContinuationToken> >(lastcontinuation);
currentRange = deserializedToken[0].Range;
Assert.AreEqual(pkRangesCount, deserializedToken.Count);
if (responseMessage.IsSuccessStatusCode)
{
Collection <ToDoActivity> response = TestCommon.Serializer.FromStream <CosmosFeedResponseUtil <ToDoActivity> >(responseMessage.Content).Data;
totalCount += response.Count;
}
if (!currentRange.Equals(previousRange))
{
visitedPkRanges++;
}
if (visitedPkRanges == pkRangesCount && responseMessage.StatusCode == System.Net.HttpStatusCode.NotModified)
{
break;
}
previousRange = currentRange;
}
}
Assert.AreEqual(firstRunTotal, totalCount);
int expectedFinalCount = 50;
previousRange = null;
currentRange = null;
visitedPkRanges = 0;
// Insert another batch of 25 and use the last continuation token from the first cycle
await this.CreateRandomItems(batchSize, randomPartitionKey : true);
FeedIterator setIteratorNew =
itemsCore.GetStandByFeedIterator(lastcontinuation);
while (setIteratorNew.HasMoreResults)
{
using (ResponseMessage responseMessage =
await setIteratorNew.ReadNextAsync(this.cancellationToken))
{
lastcontinuation = responseMessage.Headers.ContinuationToken;
Assert.AreEqual(responseMessage.ContinuationToken, responseMessage.Headers.ContinuationToken);
currentRange = JsonConvert.DeserializeObject <List <CompositeContinuationToken> >(lastcontinuation)[0].Range;
if (responseMessage.IsSuccessStatusCode)
{
Collection <ToDoActivity> response = TestCommon.Serializer.FromStream <CosmosFeedResponseUtil <ToDoActivity> >(responseMessage.Content).Data;
totalCount += response.Count;
}
if (!currentRange.Equals(previousRange))
{
visitedPkRanges++;
}
if (visitedPkRanges == pkRangesCount && responseMessage.StatusCode == System.Net.HttpStatusCode.NotModified)
{
break;
}
previousRange = currentRange;
}
}
Assert.AreEqual(expectedFinalCount, totalCount);
}
public async Task ContainerPartitionResourcePermissionTest(ConnectionMode connectionMode)
{
CosmosClientOptions cosmosClientOptions = new CosmosClientOptions()
{
ConnectionMode = connectionMode
};
CosmosClient cosmosClient = TestCommon.CreateCosmosClient(cosmosClientOptions);
Database database = await cosmosClient.CreateDatabaseIfNotExistsAsync("PermissionTest");
//create user
string userId = Guid.NewGuid().ToString();
UserResponse userResponse = await database.CreateUserAsync(userId);
Assert.AreEqual(HttpStatusCode.Created, userResponse.StatusCode);
Assert.AreEqual(userId, userResponse.Resource.Id);
User user = userResponse.User;
//create resource
string containerId = Guid.NewGuid().ToString();
ContainerResponse containerResponse = await database.CreateContainerAsync(
id : containerId,
partitionKeyPath : "/id");
Assert.AreEqual(HttpStatusCode.Created, containerResponse.StatusCode);
Container container = containerResponse.Container;
// Create items to read
ToDoActivity itemAccess = ToDoActivity.CreateRandomToDoActivity();
ToDoActivity itemNoAccess = ToDoActivity.CreateRandomToDoActivity();
await container.CreateItemAsync <ToDoActivity>(
itemAccess,
new PartitionKey(itemAccess.id));
await container.CreateItemAsync <ToDoActivity>(
itemNoAccess,
new PartitionKey(itemNoAccess.id));
//create permission
string permissionId = Guid.NewGuid().ToString();
PartitionKey partitionKey = new PartitionKey(itemAccess.id);
PermissionProperties permissionProperties = new PermissionProperties(
permissionId,
PermissionMode.Read,
container,
partitionKey);
PermissionResponse permissionResponse = await user.CreatePermissionAsync(permissionProperties);
PermissionProperties permission = permissionResponse.Resource;
Assert.AreEqual(HttpStatusCode.Created, userResponse.StatusCode);
Assert.AreEqual(permissionId, permission.Id);
Assert.AreEqual(permissionProperties.PermissionMode, permission.PermissionMode);
using (CosmosClient tokenCosmosClient = TestCommon.CreateCosmosClient(clientOptions: cosmosClientOptions, resourceToken: permission.Token))
{
Container tokenContainer = tokenCosmosClient.GetContainer(database.Id, containerId);
await tokenContainer.ReadItemAsync <ToDoActivity>(itemAccess.id, new PartitionKey(itemAccess.id));
try
{
await tokenContainer.ReadItemAsync <ToDoActivity>(itemNoAccess.id, new PartitionKey(itemNoAccess.id));
Assert.Fail();
}
catch (CosmosException ex)
{
Assert.AreEqual(HttpStatusCode.Forbidden, ex.StatusCode);
}
QueryRequestOptions queryRequestOptions = new QueryRequestOptions()
{
PartitionKey = new PartitionKey(itemAccess.id)
};
FeedIterator <ToDoActivity> feedIterator = tokenContainer.GetItemQueryIterator <ToDoActivity>(
queryText: "select * from T",
requestOptions: queryRequestOptions);
List <ToDoActivity> result = new List <ToDoActivity>();
while (feedIterator.HasMoreResults)
{
FeedResponse <ToDoActivity> toDoActivities = await feedIterator.ReadNextAsync();
result.AddRange(toDoActivities);
}
Assert.AreEqual(1, result.Count);
// Test query with no service interop via gateway query plan to replicate x32 app
ContainerCore containerCore = (ContainerInlineCore)tokenContainer;
CrossPartitionQueryTests.MockCosmosQueryClient mock = new CrossPartitionQueryTests.MockCosmosQueryClient(
clientContext: containerCore.ClientContext,
cosmosContainerCore: containerCore,
forceQueryPlanGatewayElseServiceInterop: true);
Container tokenGatewayQueryPlan = new ContainerCore(
containerCore.ClientContext,
(DatabaseCore)containerCore.Database,
containerCore.Id,
mock);
FeedIterator <ToDoActivity> feedIteratorGateway = tokenGatewayQueryPlan.GetItemQueryIterator <ToDoActivity>(
queryText: "select * from T",
requestOptions: queryRequestOptions);
List <ToDoActivity> resultGateway = new List <ToDoActivity>();
while (feedIteratorGateway.HasMoreResults)
{
FeedResponse <ToDoActivity> toDoActivities = await feedIteratorGateway.ReadNextAsync();
resultGateway.AddRange(toDoActivities);
}
Assert.AreEqual(1, resultGateway.Count);
}
}
public async Task ItemResourcePermissionTest()
{
//create user
string userId = Guid.NewGuid().ToString();
UserResponse userResponse = await this.cosmosDatabase.CreateUserAsync(userId);
Assert.AreEqual(HttpStatusCode.Created, userResponse.StatusCode);
Assert.AreEqual(userId, userResponse.Resource.Id);
User user = userResponse.User;
//create resource
string containerId = Guid.NewGuid().ToString();
ContainerResponse containerResponse = await this.cosmosDatabase.CreateContainerAsync(containerId, "/id");
Assert.AreEqual(HttpStatusCode.Created, containerResponse.StatusCode);
Container container = containerResponse.Container;
string itemId = Guid.NewGuid().ToString();
PartitionKey partitionKey = new PartitionKey(itemId);
ItemResponse <dynamic> itemRespnose = await container.CreateItemAsync <dynamic>(new { id = itemId }, partitionKey);
Assert.AreEqual(HttpStatusCode.Created, itemRespnose.StatusCode);
//create permission
string permissionId = Guid.NewGuid().ToString();
PermissionProperties permissionProperties = new PermissionProperties(permissionId, PermissionMode.Read, container, partitionKey, itemId);
PermissionResponse permissionResponse = await user.CreatePermissionAsync(permissionProperties);
PermissionProperties permission = permissionResponse.Resource;
Assert.AreEqual(HttpStatusCode.Created, userResponse.StatusCode);
Assert.AreEqual(permissionId, permission.Id);
Assert.AreEqual(permissionProperties.PermissionMode, permission.PermissionMode);
//delete resource with PermissionMode.Read
using (CosmosClient tokenCosmosClient = TestCommon.CreateCosmosClient(clientOptions: null, resourceToken: permission.Token))
{
Container tokenContainer = tokenCosmosClient.GetContainer(this.cosmosDatabase.Id, containerId);
ItemResponse <dynamic> readPermissionItem = await tokenContainer.ReadItemAsync <dynamic>(itemId, partitionKey);
Assert.AreEqual(itemId, readPermissionItem.Resource.id.ToString());
try
{
ItemResponse <dynamic> response = await tokenContainer.DeleteItemAsync <dynamic>(
itemId,
partitionKey);
Assert.Fail();
}
catch (CosmosException ex)
{
Assert.AreEqual(HttpStatusCode.Forbidden, ex.StatusCode);
}
}
//update permission to PermissionMode.All
permissionProperties = new PermissionProperties(permissionId, PermissionMode.All, container);
permissionResponse = await user.GetPermission(permissionId).ReplaceAsync(permissionProperties);
permission = permissionResponse.Resource;
//delete resource with PermissionMode.All
using (CosmosClient tokenCosmosClient = TestCommon.CreateCosmosClient(clientOptions: null, resourceToken: permission.Token))
{
FeedIterator <dynamic> feed = tokenCosmosClient
.GetDatabase(this.cosmosDatabase.Id)
.GetContainer(containerId)
.GetItemQueryIterator <dynamic>(new QueryDefinition("select * from t"));
while (feed.HasMoreResults)
{
FeedResponse <dynamic> response = await feed.ReadNextAsync();
Assert.IsNotNull(response);
}
}
}
/// <summary>Gets the user roles and permissions.</summary>
/// <param name="userDetails">The user details.</param>
/// <returns></returns>
/// <exception cref="Exception">The user could not be found in the system database</exception>
public async Task <DbModelUserRolesPermissions> GetUserRolesPermissions(UserDetails userDetails)
{
_database = await this._cosmosDbClient.CreateDatabaseIfNotExistsAsync(ServiceConfigs.CosmosDbDatabaseId);
var activeCache = await GetActiveCache(userDetails.UserPrincipalName, _database);
// return the cached user permissions object
if (activeCache != null)
{
return(activeCache);
}
// Read Users
_container = await this._database.CreateContainerIfNotExistsAsync(ServiceConfigs.UsersContainerId, Selector);
var usersQry = new QueryDefinition("SELECT TOP 1 * FROM c where c.key = @key order by c.created desc");
usersQry.WithParameter("@key", userDetails.UserPrincipalName);
FeedIterator <DbModelUser> queryResultUsersSetIterator = this._container.GetItemQueryIterator <DbModelUser>(usersQry);
var usersList = new List <DbModelUser>();
while (queryResultUsersSetIterator.HasMoreResults)
{
FeedResponse <DbModelUser> currentRolesResultSet = await queryResultUsersSetIterator.ReadNextAsync();
usersList.AddRange(currentRolesResultSet);
}
// Read user Roles by Key
if (!usersList.Any())
{
throw new Exception("The user could not be found in the system database");
}
var qryUser = usersList.First();
var newCacheItem = new DbModelUserRolesPermissions {
Id = Guid.NewGuid().ToString(), Key = qryUser.Key, Roles = qryUser.Roles, Created = DateTime.Now, Permissions = new List <BaseModel>()
};
if (qryUser.Roles.Count <= 0)
{
return(newCacheItem);
}
var filterQry = string.Empty;
foreach (var role in qryUser.Roles)
{
filterQry += (string.IsNullOrEmpty(filterQry) ? $"'{role.Key}'" : $", '{role.Key}'");
}
var rolesQry = new QueryDefinition($"SELECT * FROM c where c.key in ({filterQry})");
// Select only roles where role.Key equal to any of the user roles
_container = await this._database.CreateContainerIfNotExistsAsync(ServiceConfigs.RolesContainerId, Selector);
FeedIterator <DbModelRole> queryResultRolesSetIterator = this._container.GetItemQueryIterator <DbModelRole>(rolesQry);
var rolesList = new List <DbModelRole>();
while (queryResultRolesSetIterator.HasMoreResults)
{
FeedResponse <DbModelRole> currentRolesResultSet = await queryResultRolesSetIterator.ReadNextAsync();
rolesList.AddRange(currentRolesResultSet);
}
if (!rolesList.Any())
{
return(newCacheItem);
}
foreach (var role in qryUser.Roles)
{
var roleDefinition = rolesList.FirstOrDefault(r => r.Key == role.Key);
if (roleDefinition == null)
{
continue;
}
foreach (var permission in roleDefinition.Permissions)
{
newCacheItem.Permissions.Add(new BaseModel {
Key = permission.Key
});
}
}
newCacheItem.Permissions = newCacheItem.Permissions.Distinct().ToList();
// Store in Cache Table
_container = await this._database.CreateContainerIfNotExistsAsync(ServiceConfigs.UserRolesPermissionsContainerId, Selector);
ItemResponse <DbModelUserRolesPermissions> newCacheItemResponse = await _container.CreateItemAsync(newCacheItem);
return(newCacheItemResponse.StatusCode == HttpStatusCode.Created ? newCacheItem : null);
}
internal static async Task <List <T> > QueryWithoutContinuationTokensAsync <T>(
Container container,
string query,
QueryRequestOptions queryRequestOptions = null)
{
if (queryRequestOptions == null)
{
queryRequestOptions = new QueryRequestOptions();
}
List <T> results = new List <T>();
FeedIterator <T> itemQuery = container.GetItemQueryIterator <T>(
queryText: query,
requestOptions: queryRequestOptions);
try
{
string continuationTokenForRetries = null;
while (itemQuery.HasMoreResults)
{
try
{
FeedResponse <T> page = await itemQuery.ReadNextAsync();
results.AddRange(page);
if (queryRequestOptions.MaxItemCount.HasValue)
{
Assert.IsTrue(
page.Count <= queryRequestOptions.MaxItemCount.Value,
"Max Item Count is not being honored");
}
try
{
continuationTokenForRetries = page.ContinuationToken;
}
catch (Exception)
{
// Grabbing a continuation token is not supported on all queries.
}
}
catch (CosmosException cosmosException) when(cosmosException.StatusCode == (HttpStatusCode)429)
{
itemQuery.Dispose();
itemQuery = container.GetItemQueryIterator <T>(
queryText: query,
requestOptions: queryRequestOptions,
continuationToken: continuationTokenForRetries);
if (continuationTokenForRetries == null)
{
// The query failed and we don't have a save point, so just restart the whole thing.
results = new List <T>();
}
}
}
}
finally
{
itemQuery.Dispose();
}
return(results);
}
public async Task TestDistinct_ExecuteNextAsync()
{
async Task ImplementationAsync(Container container, IReadOnlyList <CosmosObject> documents)
{
#region Queries
// To verify distint queries you can run it once without the distinct clause and run it through a hash set
// then compare to the query with the distinct clause.
List <string> queries = new List <string>()
{
// basic distinct queries
"SELECT {0} VALUE null",
// number value distinct queries
"SELECT {0} VALUE c.income from c",
// string value distinct queries
"SELECT {0} VALUE c.name from c",
// array value distinct queries
"SELECT {0} VALUE c.children from c",
// object value distinct queries
"SELECT {0} VALUE c.pet from c",
// scalar expressions distinct query
"SELECT {0} VALUE c.age % 2 FROM c",
// distinct queries with order by
"SELECT {0} VALUE c.age FROM c ORDER BY c.age",
// distinct queries with top and no matching order by
"SELECT {0} TOP 2147483647 VALUE c.age FROM c",
// distinct queries with top and matching order by
"SELECT {0} TOP 2147483647 VALUE c.age FROM c ORDER BY c.age",
// distinct queries with aggregates
"SELECT {0} VALUE MAX(c.age) FROM c",
// distinct queries with joins
"SELECT {0} VALUE c.age FROM p JOIN c IN p.children",
// distinct queries in subqueries
"SELECT {0} r.age, s FROM r JOIN (SELECT DISTINCT VALUE c FROM (SELECT 1 a) c) s WHERE r.age > 25",
// distinct queries in scalar subqeries
"SELECT {0} p.name, (SELECT DISTINCT VALUE p.age) AS Age FROM p",
// select *
"SELECT {0} * FROM c",
};
#endregion
#region ExecuteNextAsync API
// run the query with distinct and without + MockDistinctMap
// Should receive same results
// PageSize = 1 guarantees that the backend will return some duplicates.
foreach (string query in queries)
{
string queryWithoutDistinct = string.Format(query, "");
QueryRequestOptions requestOptions = new QueryRequestOptions()
{
MaxItemCount = 100, MaxConcurrency = 100
};
FeedIterator <CosmosElement> documentQueryWithoutDistinct = container.GetItemQueryIterator <CosmosElement>(
queryWithoutDistinct,
requestOptions: requestOptions);
MockDistinctMap documentsSeen = new MockDistinctMap();
List <CosmosElement> documentsFromWithoutDistinct = new List <CosmosElement>();
while (documentQueryWithoutDistinct.HasMoreResults)
{
FeedResponse <CosmosElement> cosmosQueryResponse = await documentQueryWithoutDistinct.ReadNextAsync();
foreach (CosmosElement document in cosmosQueryResponse)
{
if (documentsSeen.Add(document, out Cosmos.Query.Core.UInt128 hash))
{
documentsFromWithoutDistinct.Add(document);
}
else
{
// No Op for debugging purposes.
}
}
}
foreach (int pageSize in new int[] { 1, 10, 100 })
{
string queryWithDistinct = string.Format(query, "DISTINCT");
List <CosmosElement> documentsFromWithDistinct = new List <CosmosElement>();
FeedIterator <CosmosElement> documentQueryWithDistinct = container.GetItemQueryIterator <CosmosElement>(
queryWithDistinct,
requestOptions: requestOptions);
while (documentQueryWithDistinct.HasMoreResults)
{
FeedResponse <CosmosElement> cosmosQueryResponse = await documentQueryWithDistinct.ReadNextAsync();
documentsFromWithDistinct.AddRange(cosmosQueryResponse);
}
Assert.AreEqual(documentsFromWithDistinct.Count, documentsFromWithoutDistinct.Count());
for (int i = 0; i < documentsFromWithDistinct.Count; i++)
{
CosmosElement documentFromWithDistinct = documentsFromWithDistinct.ElementAt(i);
CosmosElement documentFromWithoutDistinct = documentsFromWithoutDistinct.ElementAt(i);
Assert.AreEqual(
expected: documentFromWithoutDistinct,
actual: documentFromWithDistinct,
message: $"{documentFromWithDistinct} did not match {documentFromWithoutDistinct} at index {i} for {queryWithDistinct}, with page size: {pageSize} on a container");
}
}
}
#endregion
}
await this.TestQueryDistinctBaseAsync(ImplementationAsync);
}
public async Task UserDefinedFunctionsIteratorTest()
{
using (CosmosClient cosmosClient = TestCommon.CreateCosmosClient(new CosmosClientOptions()
{
Serializer = new FaultySerializer()
}))
{
// Should not use the custom serializer for these operations
Scripts scripts = cosmosClient.GetContainer(this.database.Id, this.container.Id).Scripts;
UserDefinedFunctionProperties cosmosUserDefinedFunction = await this.CreateRandomUdf();
HashSet <string> settings = new HashSet <string>();
FeedIterator <UserDefinedFunctionProperties> iter = scripts.GetUserDefinedFunctionQueryIterator <UserDefinedFunctionProperties>();;
while (iter.HasMoreResults)
{
foreach (UserDefinedFunctionProperties storedProcedureSettingsEntry in await iter.ReadNextAsync())
{
settings.Add(storedProcedureSettingsEntry.Id);
}
}
Assert.IsTrue(settings.Contains(cosmosUserDefinedFunction.Id), "The iterator did not return the user defined function definition.");
// Delete existing user defined functions.
await scripts.DeleteUserDefinedFunctionAsync(cosmosUserDefinedFunction.Id);
}
}
public async Task RunAsync(
CTLConfig config,
CosmosClient cosmosClient,
ILogger logger,
IMetrics metrics,
string loggingContextIdentifier,
CancellationToken cancellationToken)
{
logger.LogInformation("Initializing counters and metrics.");
CounterOptions documentCounter = new CounterOptions {
Name = "#Documents received", Context = loggingContextIdentifier
};
GaugeOptions leaseGauge = new GaugeOptions {
Name = "#Leases created", Context = loggingContextIdentifier
};
string leaseContainerId = Guid.NewGuid().ToString();
Container leaseContainer = await cosmosClient.GetDatabase(config.Database).CreateContainerAsync(leaseContainerId, "/id");
logger.LogInformation("Created lease container {0}", leaseContainerId);
try
{
ChangeFeedProcessor changeFeedProcessor = cosmosClient.GetContainer(config.Database, config.Collection)
.GetChangeFeedProcessorBuilder <SimpleItem>("ctlProcessor",
(IReadOnlyCollection <SimpleItem> docs, CancellationToken token) =>
{
metrics.Measure.Counter.Increment(documentCounter, docs.Count);
return(Task.CompletedTask);
})
.WithLeaseContainer(leaseContainer)
.WithInstanceName(Guid.NewGuid().ToString())
.WithStartTime(DateTime.MinValue.ToUniversalTime())
.Build();
await changeFeedProcessor.StartAsync();
logger.LogInformation("Started change feed processor");
await Task.Delay(config.RunningTimeDurationAsTimespan, cancellationToken);
logger.LogInformation("Stopping change feed processor");
await changeFeedProcessor.StopAsync();
// List leases
using FeedIterator <LeaseSchema> leaseIterator = leaseContainer.GetItemQueryIterator <LeaseSchema>();
int leaseTotal = 0;
List <FeedRange> ranges = new List <FeedRange>();
while (leaseIterator.HasMoreResults)
{
FeedResponse <LeaseSchema> response = await leaseIterator.ReadNextAsync();
foreach (LeaseSchema lease in response)
{
if (lease.LeaseToken != null)
{
logger.LogInformation($"Lease for range {lease.LeaseToken} - {lease.FeedRange.EffectiveRange.Min} - {lease.FeedRange.EffectiveRange.Max}");
ranges.Add(lease.FeedRange.EffectiveRange);
leaseTotal++;
}
}
}
string previousMin = "";
foreach (FeedRange sortedRange in ranges.OrderBy(range => range.Min))
{
if (previousMin != sortedRange.Min)
{
logger.LogError($"Expected a sorted range with Min <{previousMin}> but encountered range <{sortedRange.Min}>:<{sortedRange.Max}>");
}
previousMin = sortedRange.Max;
}
metrics.Measure.Gauge.SetValue(leaseGauge, leaseTotal);
}
catch (Exception ex)
{
logger.LogError(ex, "Failure during Change Feed Processor initialization");
}
finally
{
await leaseContainer.DeleteContainerAsync();
if (this.initializationResult.CreatedDatabase)
{
await cosmosClient.GetDatabase(config.Database).DeleteAsync();
}
if (this.initializationResult.CreatedContainer)
{
await cosmosClient.GetContainer(config.Database, config.Collection).DeleteContainerAsync();
}
}
}
public async Task ItemLINQQueryWithContinuationTokenTest()
{
// Creating items for query.
IList <ToDoActivity> itemList = await ToDoActivity.CreateRandomItems(
container : this.Container,
pkCount : 10,
perPKItemCount : 1,
randomPartitionKey : true);
IList <ToDoActivity> filteredList = itemList.Where(item => item.taskNum < 100).ToList();
int filteredDocumentCount = filteredList.Count();
Console.WriteLine($"Filtered List: {JsonConvert.SerializeObject(filteredList)}.");
QueryRequestOptions queryRequestOptions = new QueryRequestOptions();
queryRequestOptions.MaxConcurrency = 1;
queryRequestOptions.MaxItemCount = 5;
IOrderedQueryable <ToDoActivity> linqQueryable = this.Container.GetItemLinqQueryable <ToDoActivity>(requestOptions: queryRequestOptions);
IQueryable <ToDoActivity> queriable = linqQueryable.Where(item => item.taskNum < 100);
FeedIterator <ToDoActivity> feedIterator = queriable.ToFeedIterator();
int firstItemSet = 0;
string continuationToken = null;
while (feedIterator.HasMoreResults)
{
FeedResponse <ToDoActivity> feedResponse = await feedIterator.ReadNextAsync();
firstItemSet = feedResponse.Count();
continuationToken = feedResponse.ContinuationToken;
Console.WriteLine($"First page: {JsonConvert.SerializeObject(feedResponse.Resource)}.");
if (firstItemSet > 0)
{
break;
}
}
linqQueryable = this.Container.GetItemLinqQueryable <ToDoActivity>(
continuationToken: continuationToken,
requestOptions: queryRequestOptions);
queriable = linqQueryable.Where(item => item.taskNum < 100);
feedIterator = queriable.ToFeedIterator();
// Test continuationToken with LINQ query generation and asynchronous feedIterator execution.
int secondItemSet = 0;
while (feedIterator.HasMoreResults)
{
FeedResponse <ToDoActivity> feedResponse = await feedIterator.ReadNextAsync();
secondItemSet += feedResponse.Count();
Console.WriteLine($"Second Async page: {JsonConvert.SerializeObject(feedResponse.Resource)}.");
}
Assert.AreEqual(
filteredDocumentCount - firstItemSet,
secondItemSet,
"Failed to resume execution for async iterator.");
// Test continuationToken with blocking LINQ execution
linqQueryable = this.Container.GetItemLinqQueryable <ToDoActivity>(
allowSynchronousQueryExecution: true,
continuationToken: continuationToken,
requestOptions: queryRequestOptions);
List <ToDoActivity> secondSyncPage = linqQueryable.Where(item => item.taskNum < 100).ToList();
Console.WriteLine($"Second Sync page: {JsonConvert.SerializeObject(secondSyncPage)}.");
int linqExecutionItemCount = secondSyncPage.Count();
Assert.AreEqual(
filteredDocumentCount - firstItemSet,
linqExecutionItemCount,
"Failed to resume execution for sync iterator");
}
public async Task ItemTest(bool directMode)
{
CosmosClient client = directMode ? DirectCosmosClient : GatewayCosmosClient;
Container container = client.GetContainer(DatabaseId, ContainerId);
List <string> createdIds = new List <string>()
{
"BasicQueryItem",
"BasicQueryItem2",
"BasicQueryItem3"
};
List <dynamic> queryResults = await this.ToListAsync(
container.GetItemQueryStreamIterator,
container.GetItemQueryIterator <dynamic>,
"select * from T where STARTSWITH(T.id, \"BasicQueryItem\")",
CosmosBasicQueryTests.RequestOptions);
if (queryResults.Count < 3)
{
foreach (string id in createdIds)
{
dynamic item = new
{
id = id,
pk = id,
};
ItemResponse <dynamic> createResponse = await container.CreateItemAsync <dynamic>(item : item);
}
queryResults = await this.ToListAsync(
container.GetItemQueryStreamIterator,
container.GetItemQueryIterator <dynamic>,
"select * from T where STARTSWITH(T.id, \"BasicQueryItem\")",
CosmosBasicQueryTests.RequestOptions);
}
List <string> ids = queryResults.Select(x => (string)x.id).ToList();
CollectionAssert.AreEquivalent(createdIds, ids);
//Read All
List <dynamic> results = await this.ToListAsync(
container.GetItemQueryStreamIterator,
container.GetItemQueryIterator <dynamic>,
null,
CosmosBasicQueryTests.RequestOptions);
ids = results.Select(x => (string)x.id).ToList();
CollectionAssert.IsSubsetOf(createdIds, ids);
//Read All with partition key
results = await this.ToListAsync(
container.GetItemQueryStreamIterator,
container.GetItemQueryIterator <dynamic>,
null,
new QueryRequestOptions()
{
MaxItemCount = 1,
PartitionKey = new PartitionKey("BasicQueryItem")
});
Assert.AreEqual(1, results.Count);
//Read All with partition key
results = container.GetItemLinqQueryable <dynamic>(
allowSynchronousQueryExecution: true,
requestOptions: new QueryRequestOptions()
{
MaxItemCount = 1,
PartitionKey = new PartitionKey("BasicQueryItem")
}).ToList();
Assert.AreEqual(1, results.Count);
// LINQ to feed iterator Read All with partition key
FeedIterator <dynamic> iterator = container.GetItemLinqQueryable <dynamic>(
allowSynchronousQueryExecution: true,
requestOptions: new QueryRequestOptions()
{
MaxItemCount = 1,
PartitionKey = new PartitionKey("BasicQueryItem")
}).ToFeedIterator();
List <dynamic> linqResults = new List <dynamic>();
while (iterator.HasMoreResults)
{
linqResults.AddRange(await iterator.ReadNextAsync());
}
Assert.AreEqual(1, linqResults.Count);
Assert.AreEqual("BasicQueryItem", linqResults.First().pk.ToString());
}
public async Task ItemLinqReadFeedTest(bool useStatelessIterator)
{
IList <ToDoActivity> deleteList = await ToDoActivity.CreateRandomItems(this.Container, pkCount : 3, randomPartitionKey : true);
HashSet <string> itemIds = deleteList.Select(x => x.id).ToHashSet <string>();
QueryRequestOptions requestOptions = new QueryRequestOptions()
{
MaxItemCount = 1
};
List <ToDoActivity> itemsViaReadFeed = this.Container.GetItemLinqQueryable <ToDoActivity>(
allowSynchronousQueryExecution: true,
requestOptions: requestOptions).ToList();
Assert.IsTrue(itemsViaReadFeed.Count >= 3);
CollectionAssert.AreEqual(deleteList.ToList(), itemsViaReadFeed);
string lastContinuationToken = null;
FeedIterator <ToDoActivity> feedIterator = this.Container.GetItemLinqQueryable <ToDoActivity>(
requestOptions: requestOptions).ToFeedIterator();
while (feedIterator.HasMoreResults)
{
if (useStatelessIterator)
{
feedIterator = this.Container.GetItemLinqQueryable <ToDoActivity>(
continuationToken: lastContinuationToken,
requestOptions: requestOptions).ToFeedIterator();
}
FeedResponse <ToDoActivity> responseMessage = await feedIterator.ReadNextAsync(this.cancellationToken);
lastContinuationToken = responseMessage.ContinuationToken;
foreach (ToDoActivity toDoActivity in responseMessage)
{
if (itemIds.Contains(toDoActivity.id))
{
itemIds.Remove(toDoActivity.id);
}
}
}
Assert.IsNull(lastContinuationToken);
Assert.AreEqual(itemIds.Count, 0);
itemIds = deleteList.Select(x => x.id).ToHashSet <string>();
FeedIterator streamIterator = this.Container.GetItemLinqQueryable <ToDoActivity>(
requestOptions: requestOptions).ToStreamIterator();
while (streamIterator.HasMoreResults)
{
if (useStatelessIterator)
{
streamIterator = this.Container.GetItemLinqQueryable <ToDoActivity>(
continuationToken: lastContinuationToken,
requestOptions: requestOptions).ToStreamIterator();
}
using (ResponseMessage responseMessage = await streamIterator.ReadNextAsync(this.cancellationToken))
{
lastContinuationToken = responseMessage.Headers.ContinuationToken;
Collection <ToDoActivity> items = TestCommon.Serializer.FromStream <CosmosFeedResponseUtil <ToDoActivity> >(responseMessage.Content).Data;
foreach (ToDoActivity toDoActivity in items)
{
if (itemIds.Contains(toDoActivity.id))
{
itemIds.Remove(toDoActivity.id);
}
}
}
}
Assert.IsNull(lastContinuationToken);
Assert.AreEqual(itemIds.Count, 0);
}
private async Task <List <T> > ToListAsync <T>(
QueryStream createStreamQuery,
Query <T> createQuery,
string queryText,
QueryRequestOptions requestOptions)
{
HttpStatusCode expectedStatus = HttpStatusCode.OK;
FeedIterator feedStreamIterator = createStreamQuery(queryText, null, requestOptions);
List <T> streamResults = new List <T>();
while (feedStreamIterator.HasMoreResults)
{
ResponseMessage response = await feedStreamIterator.ReadNextAsync();
response.EnsureSuccessStatusCode();
Assert.AreEqual(expectedStatus, response.StatusCode);
StreamReader sr = new StreamReader(response.Content);
string result = await sr.ReadToEndAsync();
ICollection <T> responseResults = JsonConvert.DeserializeObject <CosmosFeedResponseUtil <T> >(result).Data;
Assert.IsTrue(responseResults.Count <= 1);
streamResults.AddRange(responseResults);
}
string continuationToken = null;
List <T> pagedStreamResults = new List <T>();
do
{
FeedIterator pagedFeedIterator = createStreamQuery(queryText, continuationToken, requestOptions);
ResponseMessage response = await pagedFeedIterator.ReadNextAsync();
response.EnsureSuccessStatusCode();
Assert.AreEqual(expectedStatus, response.StatusCode);
ICollection <T> responseResults = TestCommon.Serializer.FromStream <CosmosFeedResponseUtil <T> >(response.Content).Data;
Assert.IsTrue(responseResults.Count <= 1);
pagedStreamResults.AddRange(responseResults);
continuationToken = response.Headers.ContinuationToken;
Assert.AreEqual(response.ContinuationToken, response.Headers.ContinuationToken);
} while (continuationToken != null);
Assert.AreEqual(pagedStreamResults.Count, streamResults.Count);
// Both lists should be the same if not PermssionsProperties. PermissionProperties will have a different ResouceToken in the payload when read.
string streamResultString = JsonConvert.SerializeObject(streamResults);
string streamPagedResultString = JsonConvert.SerializeObject(pagedStreamResults);
if (typeof(T) != typeof(PermissionProperties))
{
Assert.AreEqual(streamPagedResultString, streamResultString);
}
FeedIterator <T> feedIterator = createQuery(queryText, null, requestOptions);
List <T> results = new List <T>();
while (feedIterator.HasMoreResults)
{
FeedResponse <T> response = await feedIterator.ReadNextAsync();
Assert.AreEqual(expectedStatus, response.StatusCode);
Assert.IsTrue(response.Count <= 1);
Assert.IsTrue(response.Resource.Count() <= 1);
results.AddRange(response);
}
continuationToken = null;
List <T> pagedResults = new List <T>();
do
{
FeedIterator <T> pagedFeedIterator = createQuery(queryText, continuationToken, requestOptions);
FeedResponse <T> response = await pagedFeedIterator.ReadNextAsync();
Assert.AreEqual(expectedStatus, response.StatusCode);
Assert.IsTrue(response.Count <= 1);
Assert.IsTrue(response.Resource.Count() <= 1);
pagedResults.AddRange(response);
continuationToken = response.ContinuationToken;
} while (continuationToken != null);
Assert.AreEqual(pagedResults.Count, results.Count);
// Both lists should be the same
string resultString = JsonConvert.SerializeObject(results);
string pagedResultString = JsonConvert.SerializeObject(pagedResults);
if (typeof(T) != typeof(PermissionProperties))
{
Assert.AreEqual(pagedResultString, resultString);
Assert.AreEqual(streamPagedResultString, resultString);
}
return(results);
}
public async Task TestQueryCrossPartitionWithLargeNumberOfKeysAsync()
{
int numberOfDocuments = 1000;
string partitionKey = "key";
HashSet <int> expectedPartitionKeyValues = new HashSet <int>();
List <string> documents = new List <string>();
for (int i = 0; i < numberOfDocuments; i++)
{
Document doc = new Document();
doc.SetPropertyValue(partitionKey, i);
documents.Add(doc.ToString());
expectedPartitionKeyValues.Add(i);
}
Assert.AreEqual(numberOfDocuments, expectedPartitionKeyValues.Count);
QueryCrossPartitionWithLargeNumberOfKeysArgs args = new QueryCrossPartitionWithLargeNumberOfKeysArgs()
{
NumberOfDocuments = numberOfDocuments,
PartitionKey = partitionKey,
ExpectedPartitionKeyValues = expectedPartitionKeyValues,
};
await this.CreateIngestQueryDeleteAsync <QueryCrossPartitionWithLargeNumberOfKeysArgs>(
ConnectionModes.Direct | ConnectionModes.Gateway,
CollectionTypes.SinglePartition | CollectionTypes.MultiPartition,
documents,
ImplementationAsync,
args,
"/" + partitionKey);
async Task ImplementationAsync(
Container container,
IReadOnlyList <CosmosObject> inputDocs,
QueryCrossPartitionWithLargeNumberOfKeysArgs testArgs)
{
QueryDefinition query = new QueryDefinition(
$"SELECT VALUE r.{args.PartitionKey} FROM r WHERE ARRAY_CONTAINS(@keys, r.{testArgs.PartitionKey})").WithParameter("@keys", testArgs.ExpectedPartitionKeyValues);
HashSet <int> actualPartitionKeyValues = new HashSet <int>();
FeedIterator <int> documentQuery = container.GetItemQueryIterator <int>(
queryDefinition: query,
requestOptions: new QueryRequestOptions()
{
MaxItemCount = -1, MaxConcurrency = 100
});
while (documentQuery.HasMoreResults)
{
FeedResponse <int> response = await documentQuery.ReadNextAsync();
foreach (int item in response)
{
actualPartitionKeyValues.Add(item);
}
}
Assert.IsTrue(actualPartitionKeyValues.SetEquals(args.ExpectedPartitionKeyValues));
}
}
// Async main requires c# 7.1 which is set in the csproj with the LangVersion attribute
// <Main>
public static async Task Main(string[] args)
{
IConfigurationRoot configuration = new ConfigurationBuilder()
.AddJsonFile("appSettings.json")
.Build();
string endpoint = configuration["EndPointUrl"];
if (string.IsNullOrEmpty(endpoint))
{
throw new ArgumentNullException("Please specify a valid endpoint in the appSettings.json");
}
string authKey = configuration["AuthorizationKey"];
if (string.IsNullOrEmpty(authKey) || string.Equals(authKey, "Super secret key"))
{
throw new ArgumentException("Please specify a valid AuthorizationKey in the appSettings.json");
}
// Connecting to Emulator. Change if you want a live account
CosmosClientBuilder cosmosClientBuilder = new CosmosClientBuilder(endpoint, authKey);
// Declare a JSON schema to use with the schema validation handler
var myContainerSchema = JSchema.Parse(@"{
'type': 'object',
'properties': {
'name': {'type': 'string'}
},
'required': ['name']
}");
cosmosClientBuilder.AddCustomHandlers(
new LoggingHandler(),
new ConcurrencyHandler(),
new ThrottlingHandler(),
new SchemaValidationHandler((database: "mydb", container: "mycoll2", schema: myContainerSchema))
);
CosmosClient client = cosmosClientBuilder.Build();
DatabaseResponse databaseResponse = await client.CreateDatabaseIfNotExistsAsync("mydb");
Database database = databaseResponse.Database;
ContainerResponse containerResponse = await database.CreateContainerIfNotExistsAsync("mycoll", "/id");
Container container = containerResponse.Container;
Item item = new Item()
{
Id = Guid.NewGuid().ToString(),
Name = "Test Item",
Description = "Some random test item",
Completed = false
};
// Create
await container.CreateItemAsync <Item>(item, new PartitionKey(item.Id));
item.Completed = true;
// Replace
await container.ReplaceItemAsync <Item>(item, item.Id, new PartitionKey(item.Id));
// Querying
FeedIterator <Item> query = container.GetItemQueryIterator <Item>(new QueryDefinition("SELECT * FROM c"), requestOptions: new QueryRequestOptions()
{
MaxConcurrency = 1
});
List <Item> results = new List <Item>();
while (query.HasMoreResults)
{
FeedResponse <Item> response = await query.ReadNextAsync();
results.AddRange(response.ToList());
}
// Read Item
ItemResponse <Item> cosmosItemResponse = await container.ReadItemAsync <Item>(item.Id, new PartitionKey(item.Id));
ItemRequestOptions itemRequestOptions = new ItemRequestOptions()
{
IfMatchEtag = cosmosItemResponse.ETag
};
// Concurrency
List <Task <ItemResponse <Item> > > tasks = new List <Task <ItemResponse <Item> > >
{
UpdateItemForConcurrency(container, itemRequestOptions, item),
UpdateItemForConcurrency(container, itemRequestOptions, item)
};
try
{
await Task.WhenAll(tasks);
}
catch (CosmosException ex)
{
// Verify that our custom handler caught the scenario
Debug.Assert(999.Equals(ex.SubStatusCode));
}
// Delete
await container.DeleteItemAsync <Item>(item.Id, new PartitionKey(item.Id));
// Schema validation
containerResponse = await database.CreateContainerIfNotExistsAsync("mycoll2", "/id");
container = containerResponse.Container;
// Insert an item with invalid schema
var writeSucceeded = true;
try
{
await container.CreateItemAsync(new { id = "12345" });
}
catch (InvalidItemSchemaException)
{
writeSucceeded = false;
}
Debug.Assert(!writeSucceeded);
// Insert an item with valid schema
try
{
await container.CreateItemAsync(new { id = "12345", name = "Youri" });
writeSucceeded = true;
}
catch (InvalidItemSchemaException)
{
writeSucceeded = false;
}
Debug.Assert(writeSucceeded);
// Update an item with invalid schema
try
{
await container.ReplaceItemAsync(new { id = "12345" }, "12345");
writeSucceeded = true;
}
catch (InvalidItemSchemaException)
{
writeSucceeded = false;
}
Debug.Assert(!writeSucceeded);
// Update an item with valid schema
try
{
await container.ReplaceItemAsync(new { id = "12345", name = "Vladimir" }, "12345");
writeSucceeded = true;
}
catch (InvalidItemSchemaException)
{
writeSucceeded = false;
}
Debug.Assert(writeSucceeded);
}
public async Task ChangeFeedIteratorCore_PartitionKey_OfT_ReadAll()
{
int totalCount = 0;
int firstRunTotal = 25;
int batchSize = 25;
string pkToRead = "pkToRead";
string otherPK = "otherPK";
ContainerInternal itemsCore = await this.InitializeContainerAsync();
for (int i = 0; i < batchSize; i++)
{
await itemsCore.CreateItemAsync(ToDoActivity.CreateRandomToDoActivity(pk: pkToRead));
}
for (int i = 0; i < batchSize; i++)
{
await itemsCore.CreateItemAsync(ToDoActivity.CreateRandomToDoActivity(pk: otherPK));
}
FeedIterator <ToDoActivity> feedIterator = itemsCore.GetChangeFeedIterator <ToDoActivity>(
ChangeFeedStartFrom.Beginning(
new FeedRangePartitionKey(
new PartitionKey(pkToRead))),
ChangeFeedMode.Incremental,
new ChangeFeedRequestOptions()
{
PageSizeHint = 1,
});
string continuation = null;
while (feedIterator.HasMoreResults)
{
try
{
FeedResponse <ToDoActivity> feedResponse = await feedIterator.ReadNextAsync(this.cancellationToken);
totalCount += feedResponse.Count;
foreach (ToDoActivity toDoActivity in feedResponse)
{
Assert.AreEqual(pkToRead, toDoActivity.pk);
}
continuation = feedResponse.ContinuationToken;
}
catch (CosmosException cosmosException) when(cosmosException.StatusCode == HttpStatusCode.NotModified)
{
continuation = cosmosException.Headers.ContinuationToken;
break;
}
}
Assert.AreEqual(firstRunTotal, totalCount);
int expectedFinalCount = 50;
// Insert another batch of 25 and use the last FeedToken from the first cycle
for (int i = 0; i < batchSize; i++)
{
await itemsCore.CreateItemAsync(ToDoActivity.CreateRandomToDoActivity(pk: pkToRead));
}
FeedIterator <ToDoActivity> setIteratorNew = itemsCore.GetChangeFeedIterator <ToDoActivity>(
ChangeFeedStartFrom.ContinuationToken(continuation),
ChangeFeedMode.Incremental);
while (setIteratorNew.HasMoreResults)
{
try
{
FeedResponse <ToDoActivity> feedResponse = await setIteratorNew.ReadNextAsync(this.cancellationToken);
totalCount += feedResponse.Count;
foreach (ToDoActivity toDoActivity in feedResponse)
{
Assert.AreEqual(pkToRead, toDoActivity.pk);
}
}
catch (CosmosException cosmosException) when(cosmosException.StatusCode == HttpStatusCode.NotModified)
{
break;
}
}
Assert.AreEqual(expectedFinalCount, totalCount);
}
public async Task ChangeFeedIteratorCore_PartitionKey_OfT_ReadAll()
{
int totalCount = 0;
int firstRunTotal = 25;
int batchSize = 25;
string pkToRead = "pkToRead";
string otherPK = "otherPK";
for (int i = 0; i < batchSize; i++)
{
await this.Container.CreateItemAsync(this.CreateRandomToDoActivity(pkToRead));
}
for (int i = 0; i < batchSize; i++)
{
await this.Container.CreateItemAsync(this.CreateRandomToDoActivity(otherPK));
}
ContainerCore itemsCore = this.Container;
FeedIterator <ToDoActivity> feedIterator = itemsCore.GetChangeFeedIterator <ToDoActivity>(new PartitionKey(pkToRead), changeFeedRequestOptions: new ChangeFeedRequestOptions()
{
StartTime = DateTime.MinValue.ToUniversalTime()
});
while (feedIterator.HasMoreResults)
{
FeedResponse <ToDoActivity> feedResponse = await feedIterator.ReadNextAsync(this.cancellationToken);
totalCount += feedResponse.Count;
foreach (ToDoActivity toDoActivity in feedResponse)
{
Assert.AreEqual(pkToRead, toDoActivity.status);
}
}
Assert.AreEqual(firstRunTotal, totalCount);
int expectedFinalCount = 50;
// Insert another batch of 25 and use the last FeedToken from the first cycle
for (int i = 0; i < batchSize; i++)
{
await this.Container.CreateItemAsync(this.CreateRandomToDoActivity(pkToRead));
}
FeedIterator <ToDoActivity> setIteratorNew = itemsCore.GetChangeFeedIterator <ToDoActivity>(feedToken: feedIterator.FeedToken, changeFeedRequestOptions: new ChangeFeedRequestOptions()
{
StartTime = DateTime.MinValue.ToUniversalTime()
});
while (setIteratorNew.HasMoreResults)
{
FeedResponse <ToDoActivity> feedResponse = await setIteratorNew.ReadNextAsync(this.cancellationToken);
totalCount += feedResponse.Count;
foreach (ToDoActivity toDoActivity in feedResponse)
{
Assert.AreEqual(pkToRead, toDoActivity.status);
}
}
Assert.AreEqual(expectedFinalCount, totalCount);
}
private static async Task IterateDekFeedAsync(
CosmosDataEncryptionKeyProvider dekProvider,
List <string> expectedDekIds,
bool isExpectedDeksCompleteSetForRequest,
bool isResultOrderExpected,
string query,
int?itemCountInPage = null)
{
int remainingItemCount = expectedDekIds.Count;
QueryRequestOptions requestOptions = null;
if (itemCountInPage.HasValue)
{
requestOptions = new QueryRequestOptions()
{
MaxItemCount = itemCountInPage
};
}
FeedIterator <DataEncryptionKeyProperties> dekIterator = dekProvider.DataEncryptionKeyContainer
.GetDataEncryptionKeyQueryIterator <DataEncryptionKeyProperties>(
query,
requestOptions: requestOptions);
Assert.IsTrue(dekIterator.HasMoreResults);
List <string> readDekIds = new List <string>();
while (remainingItemCount > 0)
{
FeedResponse <DataEncryptionKeyProperties> page = await dekIterator.ReadNextAsync();
if (itemCountInPage.HasValue)
{
// last page
if (remainingItemCount < itemCountInPage.Value)
{
Assert.AreEqual(remainingItemCount, page.Count);
}
else
{
Assert.AreEqual(itemCountInPage.Value, page.Count);
}
}
else
{
Assert.AreEqual(expectedDekIds.Count, page.Count);
}
remainingItemCount -= page.Count;
if (isExpectedDeksCompleteSetForRequest)
{
Assert.AreEqual(remainingItemCount > 0, dekIterator.HasMoreResults);
}
foreach (DataEncryptionKeyProperties dek in page.Resource)
{
readDekIds.Add(dek.Id);
}
}
if (isResultOrderExpected)
{
Assert.IsTrue(expectedDekIds.SequenceEqual(readDekIds));
}
else
{
Assert.IsTrue(expectedDekIds.ToHashSet().SetEquals(readDekIds));
}
}
//Helper method to run query
private async Task RunQuery(string sqlQueryText, string databaseName, string containerName, int maxItemCountPerPage = 100, int maxConcurrency = -1, bool useQueryOptions = false) //100 or 1MB, whichever comes first
{
Console.BackgroundColor = ConsoleColor.Blue;
Console.WriteLine("Running query: {0} against container {1}\n", sqlQueryText, containerName);
if (useQueryOptions)
{
Console.WriteLine("Using MaxConcurrency: {0}", maxConcurrency);
Console.WriteLine("Using MaxItemCountPerPage: {0}", maxItemCountPerPage);
}
Console.ResetColor();
double totalRequestCharge = 0;
QueryDefinition queryDefinition = new QueryDefinition(sqlQueryText);
// Run query against Cosmos DB
var container = cosmosClient.GetDatabase(databaseName).GetContainer(containerName);
QueryRequestOptions requestOptions;
if (useQueryOptions)
{
requestOptions = new QueryRequestOptions()
{
MaxItemCount = maxItemCountPerPage,
MaxConcurrency = maxConcurrency,
};
}
else
{
requestOptions = new QueryRequestOptions(); //use all default query options
}
// Time the query
Stopwatch stopWatch = new Stopwatch();
stopWatch.Start();
FeedIterator <dynamic> queryResultSetIterator = container.GetItemQueryIterator <dynamic>(queryDefinition, requestOptions: requestOptions);
List <dynamic> reviews = new List <dynamic>();
while (queryResultSetIterator.HasMoreResults)
{
FeedResponse <dynamic> currentResultSet = await queryResultSetIterator.ReadNextAsync();
totalRequestCharge += currentResultSet.RequestCharge;
//Console.WriteLine("another page");
foreach (var item in currentResultSet)
{
reviews.Add(item);
}
if (useQueryOptions)
{
Console.WriteLine("Result count: {0}", reviews.Count);
}
}
stopWatch.Stop();
TimeSpan ts = stopWatch.Elapsed;
//Print results
string elapsedTime = String.Format("{0:00}:{1:00}:{2:00}.{3:00}",
ts.Hours, ts.Minutes, ts.Seconds,
ts.Milliseconds / 10);
Console.ForegroundColor = ConsoleColor.Green;
Console.WriteLine("\tQuery returned {0} results", reviews.Count);
Console.WriteLine("\tTotal time: {0}", elapsedTime);
Console.WriteLine("\tTotal Request Units consumed: {0}\n", totalRequestCharge);
Console.WriteLine("\n\n\n");
Console.ResetColor();
}
// </RunIndexDemo>
/// <summary>
/// The default index policy on a Container will AUTOMATICALLY index ALL documents added.
/// There may be scenarios where you want to exclude a specific doc from the index even though all other
/// documents are being indexed automatically.
/// This method demonstrates how to use an index directive to control this
/// </summary>
private static async Task ExplicitlyExcludeFromIndex(CosmosClient client)
{
string containerId = $"{Program.containerId}-ExplicitlyExcludeFromIndex";
// Create a collection with default index policy(i.e.automatic = true)
ContainerResponse response = await Program.database.CreateContainerAsync(containerId, Program.partitionKey);
Console.WriteLine("Container {0} created with index policy \n{1}", containerId, JsonConvert.SerializeObject(response.Resource.IndexingPolicy));
Container container = (Container)response;
try
{
// Create a document
// Then query on it immediately
// Will work as this Collection is set to automatically index everything
ItemResponse <dynamic> created = await container.CreateItemAsync <dynamic>(new { id = "doc1", partitionKey = "doc1", orderId = "order1" }, new PartitionKey("doc1"));
Console.WriteLine("\nItem created: \n{0}", JsonConvert.SerializeObject(created.Resource));
using (FeedIterator <dynamic> resultSetIterator = container.GetItemQueryIterator <dynamic>(
new QueryDefinition("SELECT * FROM root r WHERE r.orderId='order1'"),
requestOptions: new QueryRequestOptions {
MaxItemCount = 1
}))
{
while (resultSetIterator.HasMoreResults)
{
FeedResponse <dynamic> feedResponse = await resultSetIterator.ReadNextAsync();
Console.WriteLine($"Item count by query: {feedResponse.Count}");
}
}
// Now, create an item but this time explictly exclude it from the collection using IndexingDirective
// Then query for that document
// Shoud NOT find it, because we excluded it from the index
// BUT, the document is there and doing a ReadItem by Id will prove it
created = await container.CreateItemAsync <dynamic>(new { id = "doc2", partitionKey = "doc2", orderId = "order2" }, new PartitionKey("doc2"), new ItemRequestOptions
{
IndexingDirective = IndexingDirective.Exclude
});
Console.WriteLine("\nItem created: \n{0}", JsonConvert.SerializeObject(created.Resource));
using (FeedIterator <dynamic> resultSetIterator = container.GetItemQueryIterator <dynamic>(
new QueryDefinition("SELECT * FROM root r WHERE r.orderId='order2'"),
requestOptions: new QueryRequestOptions {
MaxItemCount = 1
}))
{
while (resultSetIterator.HasMoreResults)
{
FeedResponse <dynamic> feedResponse = await resultSetIterator.ReadNextAsync();
Console.WriteLine($"Item count by query: {feedResponse.Count}");
}
}
ItemResponse <dynamic> document = await container.ReadItemAsync <dynamic>((string)created.Resource.id, new PartitionKey("doc2"));
Console.WriteLine("Item read by id: {0}", document != null);
}
finally
{
// Cleanup
await container.DeleteContainerAsync();
}
}
public async Task StandByFeedIterator()
{
int totalCount = 0;
string lastcontinuation = string.Empty;
int firstRunTotal = 25;
int batchSize = 25;
int pkRangesCount = (await this.Container.ClientContext.DocumentClient.ReadPartitionKeyRangeFeedAsync(this.Container.LinkUri)).Count;
await this.CreateRandomItems(this.Container, batchSize, randomPartitionKey : true);
ContainerInternal itemsCore = this.Container;
FeedIterator feedIterator = itemsCore.GetStandByFeedIterator(requestOptions: new StandByFeedIteratorRequestOptions()
{
StartTime = DateTime.MinValue
});
while (feedIterator.HasMoreResults)
{
using (ResponseMessage responseMessage =
await feedIterator.ReadNextAsync(this.cancellationToken))
{
lastcontinuation = responseMessage.Headers.ContinuationToken;
Assert.AreEqual(responseMessage.ContinuationToken, responseMessage.Headers.ContinuationToken);
List <CompositeContinuationToken> deserializedToken = JsonConvert.DeserializeObject <List <CompositeContinuationToken> >(lastcontinuation);
Assert.AreEqual(pkRangesCount, deserializedToken.Count);
if (!responseMessage.IsSuccessStatusCode)
{
break;
}
Collection <ToDoActivity> response = TestCommon.SerializerCore.FromStream <CosmosFeedResponseUtil <ToDoActivity> >(responseMessage.Content).Data;
totalCount += response.Count;
}
}
Assert.AreEqual(firstRunTotal, totalCount);
int expectedFinalCount = 50;
// Insert another batch of 25 and use the last continuation token from the first cycle
await this.CreateRandomItems(this.Container, batchSize, randomPartitionKey : true);
FeedIterator setIteratorNew =
itemsCore.GetStandByFeedIterator(lastcontinuation);
while (setIteratorNew.HasMoreResults)
{
using (ResponseMessage responseMessage =
await setIteratorNew.ReadNextAsync(this.cancellationToken))
{
lastcontinuation = responseMessage.Headers.ContinuationToken;
Assert.AreEqual(responseMessage.ContinuationToken, responseMessage.Headers.ContinuationToken);
if (!responseMessage.IsSuccessStatusCode)
{
break;
}
Collection <ToDoActivity> response = TestCommon.SerializerCore.FromStream <CosmosFeedResponseUtil <ToDoActivity> >(responseMessage.Content).Data;
totalCount += response.Count;
}
}
Assert.AreEqual(expectedFinalCount, totalCount);
}
// Gets alla the questionnaires from the database
public static async Task <List <KeyValuePair <string, List <QuestionTopFive> > > > QueryQuestionnairesAsync()
{
var sqlQueryText = "SELECT c.document.Key, c.document[\"Value\"] FROM c";
QueryDefinition queryDefinition = new QueryDefinition(sqlQueryText);
CosmosClient cosmosClient = new CosmosClient(cosmosServiceEndpoint, cosmosDBKey);
Azure.Cosmos.Database database;
database = await cosmosClient.CreateDatabaseIfNotExistsAsync(cosmosDBDatabaseName);
Container container;
container = await database.CreateContainerIfNotExistsAsync(cosmosDBConteinerIdQuestionnaires, "/id");
FeedIterator <KeyValuePair <string, List <QuestionTopFive> > > queryResultSetIterator = container.GetItemQueryIterator <KeyValuePair <string, List <QuestionTopFive> > >(queryDefinition);
var questionnaireList = new List <KeyValuePair <string, List <QuestionTopFive> > >();
while (queryResultSetIterator.HasMoreResults)
{
Azure.Cosmos.FeedResponse <KeyValuePair <string, List <QuestionTopFive> > > currentResultSet = await queryResultSetIterator.ReadNextAsync();
foreach (KeyValuePair <string, List <QuestionTopFive> > questionnaire in currentResultSet)
{
questionnaireList.Add(questionnaire);
}
}
return(ReplaceMissingQuestionValues(questionnaireList));
}
public async Task ItemLINQQueryTest()
{
//Creating items for query.
IList <ToDoActivity> itemList = await ToDoActivity.CreateRandomItems(container : this.Container, pkCount : 2, perPKItemCount : 1, randomPartitionKey : true);
IOrderedQueryable <ToDoActivity> linqQueryable = this.Container.GetItemLinqQueryable <ToDoActivity>();
IQueryable <ToDoActivity> queriable = linqQueryable.Where(item => item.taskNum < 100);
//V3 Asynchronous query execution with LINQ query generation sql text.
FeedIterator <ToDoActivity> setIterator = this.Container.GetItemQueryIterator <ToDoActivity>(
queriable.ToQueryDefinition(),
requestOptions: new QueryRequestOptions()
{
MaxConcurrency = 2
});
int resultsFetched = 0;
while (setIterator.HasMoreResults)
{
FeedResponse <ToDoActivity> queryResponse = await setIterator.ReadNextAsync();
resultsFetched += queryResponse.Count();
// For the items returned with NonePartitionKeyValue
IEnumerator <ToDoActivity> iter = queryResponse.GetEnumerator();
while (iter.MoveNext())
{
ToDoActivity activity = iter.Current;
Assert.AreEqual(42, activity.taskNum);
}
Assert.AreEqual(2, resultsFetched);
}
//LINQ query execution without partition key.
linqQueryable = this.Container.GetItemLinqQueryable <ToDoActivity>(allowSynchronousQueryExecution: true);
queriable = linqQueryable.Where(item => item.taskNum < 100);
Assert.AreEqual(2, queriable.Count());
Assert.AreEqual(itemList[0].id, queriable.ToList()[0].id);
Assert.AreEqual(itemList[1].id, queriable.ToList()[1].id);
//LINQ query execution with wrong partition key.
linqQueryable = this.Container.GetItemLinqQueryable <ToDoActivity>(
allowSynchronousQueryExecution: true,
requestOptions: new QueryRequestOptions()
{
PartitionKey = new Cosmos.PartitionKey("test")
});
queriable = linqQueryable.Where(item => item.taskNum < 100);
Assert.AreEqual(0, queriable.Count());
//LINQ query execution with correct partition key.
linqQueryable = this.Container.GetItemLinqQueryable <ToDoActivity>(
allowSynchronousQueryExecution: true,
requestOptions: new QueryRequestOptions {
ConsistencyLevel = Cosmos.ConsistencyLevel.Eventual, PartitionKey = new Cosmos.PartitionKey(itemList[1].status)
});
queriable = linqQueryable.Where(item => item.taskNum < 100);
Assert.AreEqual(1, queriable.Count());
Assert.AreEqual(itemList[1].id, queriable.ToList()[0].id);
}
public async Task QueryOperationDiagnostic()
{
IList <ToDoActivity> itemList = await ToDoActivity.CreateRandomItems(this.Container, 3, randomPartitionKey : true);
//Checking query metrics on typed query
ToDoActivity find = itemList.First();
QueryDefinition sql = new QueryDefinition("select * from ToDoActivity");
QueryRequestOptions requestOptions = new QueryRequestOptions()
{
MaxItemCount = 1,
MaxConcurrency = 1,
};
FeedIterator <ToDoActivity> feedIterator = this.Container.GetItemQueryIterator <ToDoActivity>(
sql,
requestOptions: requestOptions);
if (feedIterator.HasMoreResults)
{
FeedResponse <ToDoActivity> iter = await feedIterator.ReadNextAsync();
Assert.IsTrue(((QueryOperationStatistics)iter.Diagnostics).queryMetrics.Values.First().OutputDocumentCount > 0);
}
sql = new QueryDefinition("select * from ToDoActivity t ORDER BY t.cost");
feedIterator = this.Container.GetItemQueryIterator <ToDoActivity>(
sql,
requestOptions: requestOptions);
if (feedIterator.HasMoreResults)
{
FeedResponse <ToDoActivity> iter = await feedIterator.ReadNextAsync();
Assert.IsTrue(((QueryOperationStatistics)iter.Diagnostics).queryMetrics.Values.First().OutputDocumentCount > 0);
}
sql = new QueryDefinition("select DISTINCT t.cost from ToDoActivity t");
feedIterator = this.Container.GetItemQueryIterator <ToDoActivity>(
sql,
requestOptions: requestOptions);
if (feedIterator.HasMoreResults)
{
FeedResponse <ToDoActivity> iter = await feedIterator.ReadNextAsync();
Assert.IsNotNull((QueryOperationStatistics)iter.Diagnostics);
Assert.AreEqual(1, ((QueryOperationStatistics)iter.Diagnostics).queryMetrics.Values.First().OutputDocumentCount);
}
sql = new QueryDefinition("select * from ToDoActivity OFFSET 1 LIMIT 1");
feedIterator = this.Container.GetItemQueryIterator <ToDoActivity>(
sql,
requestOptions: requestOptions);
if (feedIterator.HasMoreResults)
{
FeedResponse <ToDoActivity> iter = await feedIterator.ReadNextAsync();
Assert.IsTrue(((QueryOperationStatistics)iter.Diagnostics).queryMetrics.Values.First().OutputDocumentCount > 0);
}
//Checking query metrics on stream query
sql = new QueryDefinition("select * from ToDoActivity");
FeedIterator iterator = this.Container.GetItemQueryStreamIterator(
sql,
requestOptions: requestOptions);
if (iterator.HasMoreResults)
{
ResponseMessage responseMessage = await iterator.ReadNextAsync();
Assert.IsTrue(((QueryOperationStatistics)responseMessage.Diagnostics).queryMetrics.Values.First().OutputDocumentCount > 0);
}
}
public async Task ChangeFeedIteratorCore_PartitionKey_OfT_ReadAll()
{
int totalCount = 0;
int firstRunTotal = 25;
int batchSize = 25;
string pkToRead = "pkToRead";
string otherPK = "otherPK";
for (int i = 0; i < batchSize; i++)
{
await this.Container.CreateItemAsync(this.CreateRandomToDoActivity(pkToRead));
}
for (int i = 0; i < batchSize; i++)
{
await this.Container.CreateItemAsync(this.CreateRandomToDoActivity(otherPK));
}
ContainerInternal itemsCore = this.Container;
FeedIterator <ToDoActivity> feedIterator = itemsCore.GetChangeFeedIterator <ToDoActivity>(
changeFeedRequestOptions: new ChangeFeedRequestOptions()
{
FeedRange = new FeedRangePartitionKey(new PartitionKey(pkToRead)),
From = ChangeFeedRequestOptions.StartFrom.CreateFromBeginning(),
});
string continuation = null;
while (feedIterator.HasMoreResults)
{
FeedResponse <ToDoActivity> feedResponse = await feedIterator.ReadNextAsync(this.cancellationToken);
totalCount += feedResponse.Count;
foreach (ToDoActivity toDoActivity in feedResponse)
{
Assert.AreEqual(pkToRead, toDoActivity.status);
}
continuation = feedResponse.ContinuationToken;
}
Assert.AreEqual(firstRunTotal, totalCount);
int expectedFinalCount = 50;
// Insert another batch of 25 and use the last FeedToken from the first cycle
for (int i = 0; i < batchSize; i++)
{
await this.Container.CreateItemAsync(this.CreateRandomToDoActivity(pkToRead));
}
FeedIterator <ToDoActivity> setIteratorNew = itemsCore.GetChangeFeedIterator <ToDoActivity>(
changeFeedRequestOptions: new ChangeFeedRequestOptions()
{
From = ChangeFeedRequestOptions.StartFrom.CreateFromContinuation(continuation),
});
while (setIteratorNew.HasMoreResults)
{
FeedResponse <ToDoActivity> feedResponse = await setIteratorNew.ReadNextAsync(this.cancellationToken);
totalCount += feedResponse.Count;
foreach (ToDoActivity toDoActivity in feedResponse)
{
Assert.AreEqual(pkToRead, toDoActivity.status);
}
}
Assert.AreEqual(expectedFinalCount, totalCount);
}
