public async Task IteratorTest()
{
string containerName = Guid.NewGuid().ToString();
string partitionKeyPath = "/users";
ContainerResponse containerResponse = await this.cosmosDatabase.CreateContainerAsync(containerName, partitionKeyPath);
Assert.AreEqual(HttpStatusCode.Created, containerResponse.StatusCode);
Assert.AreEqual(containerName, containerResponse.Resource.Id);
Assert.AreEqual(partitionKeyPath, containerResponse.Resource.PartitionKey.Paths.First());
HashSet <string> containerIds = new HashSet <string>();
FeedIterator <CosmosContainerSettings> resultSet = this.cosmosDatabase.GetContainersIterator();
while (resultSet.HasMoreResults)
{
foreach (CosmosContainerSettings setting in await resultSet.FetchNextSetAsync())
{
if (!containerIds.Contains(setting.Id))
{
containerIds.Add(setting.Id);
}
}
}
Assert.IsTrue(containerIds.Count > 0, "The iterator did not find any containers.");
Assert.IsTrue(containerIds.Contains(containerName), "The iterator did not find the created container");
containerResponse = await containerResponse.Container.DeleteAsync();
Assert.AreEqual(HttpStatusCode.NoContent, containerResponse.StatusCode);
}
private static async Task ItemStreamFeed(CosmosContainer container)
{
int totalCount = 0;
// SQL
FeedIterator setIterator = container.GetItemsStreamIterator();
while (setIterator.HasMoreResults)
{
int count = 0;
using (CosmosResponseMessage response = await setIterator.FetchNextSetAsync())
{
response.EnsureSuccessStatusCode();
count++;
using (StreamReader sr = new StreamReader(response.Content))
using (JsonTextReader jtr = new JsonTextReader(sr))
{
JsonSerializer jsonSerializer = new JsonSerializer();
dynamic array = jsonSerializer.Deserialize <dynamic>(jtr);
totalCount += array.Documents.Count;
}
}
}
Assert("Expected two families", totalCount == 2);
}
public async Task StandByFeedIterator_WithMaxItemCount()
{
await this.CreateRandomItems(2, randomPartitionKey : true);
CosmosContainerCore itemsCore = (CosmosContainerCore)this.Container;
FeedIterator feedIterator = itemsCore.GetStandByFeedIterator(maxItemCount: 1, requestOptions: new ChangeFeedRequestOptions()
{
StartTime = DateTime.MinValue
});
while (feedIterator.HasMoreResults)
{
using (CosmosResponseMessage responseMessage =
await feedIterator.FetchNextSetAsync(this.cancellationToken))
{
if (responseMessage.IsSuccessStatusCode)
{
Collection <ToDoActivity> response = new CosmosJsonSerializerCore().FromStream <CosmosFeedResponseUtil <ToDoActivity> >(responseMessage.Content).Data;
if (response.Count > 0)
{
Assert.AreEqual(1, response.Count);
return;
}
}
}
}
Assert.Fail("Found no batch with size 1");
}
private static async Task QueryItemsInPartitionAsStreams(CosmosContainer container)
{
// SQL
FeedIterator setIterator = container.CreateItemQueryStream(
"SELECT F.id, F.LastName, F.IsRegistered FROM Families F",
partitionKey: new PartitionKey("Anderson"),
maxConcurrency: 1,
maxItemCount: 1);
int count = 0;
while (setIterator.HasMoreResults)
{
using (CosmosResponseMessage response = await setIterator.FetchNextSetAsync())
{
Assert("Response failed", response.IsSuccessStatusCode);
count++;
using (StreamReader sr = new StreamReader(response.Content))
using (JsonTextReader jtr = new JsonTextReader(sr))
{
JsonSerializer jsonSerializer = new JsonSerializer();
dynamic items = jsonSerializer.Deserialize <dynamic>(jtr).Documents;
Assert("Expected one family", items.Count == 1);
dynamic item = items[0];
Assert($"Expected LastName: Anderson Actual: {item.LastName}", string.Equals("Anderson", item.LastName.ToString(), StringComparison.InvariantCulture));
}
}
}
Assert("Expected 1 family", count == 1);
}
private async Task TestOrderyByQueryAsync()
{
var jsonSerializerSettings = new JsonSerializerSettings
{
ConstructorHandling = ConstructorHandling.AllowNonPublicDefaultConstructor,
Converters =
{
new ObjectStringJsonConverter <SerializedObject>(_ => _.Name, _ => SerializedObject.Parse(_))
}
};
CosmosClient cosmosClient = TestCommon.CreateCosmosClient((cosmosClientBuilder) => cosmosClientBuilder.WithCustomJsonSerializer(new CustomJsonSerializer(jsonSerializerSettings)));
CosmosContainer container = cosmosClient.Databases[databaseName].Containers[partitionedCollectionName];
// Create a few test documents
int documentCount = 3;
var numberFieldName = "NumberField";
for (int i = 0; i < documentCount; ++i)
{
var newDocument = new MyObject(i);
var createdDocument = await container.CreateItemAsync <MyObject>(newDocument.pk, newDocument);
}
CosmosSqlQueryDefinition cosmosSqlQueryDefinition1 = new CosmosSqlQueryDefinition("SELECT * FROM root");
FeedIterator <MyObject> setIterator1 = container.CreateItemQuery <MyObject>(cosmosSqlQueryDefinition1, maxConcurrency: -1, maxItemCount: -1);
CosmosSqlQueryDefinition cosmosSqlQueryDefinition2 = new CosmosSqlQueryDefinition("SELECT * FROM root ORDER BY root[\"" + numberFieldName + "\"] DESC");
FeedIterator <MyObject> setIterator2 = container.CreateItemQuery <MyObject>(cosmosSqlQueryDefinition2, maxConcurrency: -1, maxItemCount: -1);
List <MyObject> list1 = new List <MyObject>();
List <MyObject> list2 = new List <MyObject>();
while (setIterator1.HasMoreResults)
{
foreach (MyObject obj in await setIterator1.FetchNextSetAsync())
{
list1.Add(obj);
}
}
while (setIterator2.HasMoreResults)
{
foreach (MyObject obj in await setIterator2.FetchNextSetAsync())
{
list2.Add(obj);
}
}
Assert.AreEqual(documentCount, list1.Count);
Assert.AreEqual(documentCount, list2.Count);
for (int i = 0; i < documentCount; ++i)
{
Assert.AreEqual("Name: " + (documentCount - i - 1), list2[i].SerializedObject.Name);
}
}
public async ValueTask <bool> MoveNextAsync()
{
_cancellationToken.ThrowIfCancellationRequested();
if (_jsonReader == null)
{
if (_query == null)
{
_query = _cosmosClient.CreateQuery(_containerId, _cosmosSqlQuery);
}
if (!_query.HasMoreResults)
{
Current = default;
return(false);
}
_responseStream = (await _query.FetchNextSetAsync(_cancellationToken)).Content;
_reader = new StreamReader(_responseStream);
_jsonReader = new JsonTextReader(_reader);
while (_jsonReader.Read())
{
if (_jsonReader.TokenType == JsonToken.StartObject)
{
while (_jsonReader.Read())
{
if (_jsonReader.TokenType == JsonToken.StartArray)
{
goto ObjectFound;
}
}
}
}
ObjectFound:
;
}
while (_jsonReader.Read())
{
if (_jsonReader.TokenType == JsonToken.StartObject)
{
Current = new JsonSerializer().Deserialize <JObject>(_jsonReader);
return(true);
}
}
_jsonReader.Close();
_jsonReader = null;
_reader.Dispose();
_reader = null;
_responseStream.Dispose();
_responseStream = null;
return(await MoveNextAsync());
}
/// <summary>
/// The function demonstrates migrating documents that were inserted without a value for partition key, and those inserted
/// pre-migration to other logical partitions, those with a value for partition key.
/// </summary>
private static async Task MigratedItemsFromNonePartitionKeyToValidPartitionKeyValue(CosmosContainer container)
{
// Pre-create a few items in the container to demo the migration
const int ItemsToCreate = 4;
// Insert a few items with no Partition Key
for (int i = 0; i < ItemsToCreate; i++)
{
string itemid = Guid.NewGuid().ToString();
DeviceInformationItem itemWithoutPK = GetDeviceWithNoPartitionKey(itemid);
ItemResponse <DeviceInformationItem> createResponse = await container.CreateItemAsync <DeviceInformationItem>(
partitionKey : PartitionKey.NonePartitionKeyValue,
item : itemWithoutPK);
}
// Query items on the container that have no partition key value by supplying NonePartitionKeyValue
// The operation is made in batches to not lose work in case of partial execution
int resultsFetched = 0;
CosmosSqlQueryDefinition sql = new CosmosSqlQueryDefinition("select * from r");
FeedIterator <DeviceInformationItem> setIterator = container.CreateItemQuery <DeviceInformationItem>(sql, partitionKey: PartitionKey.NonePartitionKeyValue, maxItemCount: 2);
while (setIterator.HasMoreResults)
{
FeedResponse <DeviceInformationItem> queryResponse = await setIterator.FetchNextSetAsync();
resultsFetched += queryResponse.Count();
// For the items returned with NonePartitionKeyValue
IEnumerator <DeviceInformationItem> iter = queryResponse.GetEnumerator();
while (iter.MoveNext())
{
DeviceInformationItem item = iter.Current;
if (item.DeviceId != null)
{
// Using existing deviceID for partition key
item.PartitionKey = item.DeviceId;
Console.WriteLine("Migrating item {0} to Partition {1}", item.Id, item.DeviceId);
// Re-Insert into container with a partition key
// This could result in exception if the same item was inserted in a previous run of the program on existing container
// and the program stopped before the delete.
ItemResponse <DeviceInformationItem> createResponseWithPk = await container.CreateItemAsync <DeviceInformationItem>(
partitionKey : new PartitionKey(item.PartitionKey),
item : item);
// Deleting item from fixed container with CosmosContainerSettings.NonePartitionKeyValue.
ItemResponse <DeviceInformationItem> deleteResponseWithoutPk = await container.DeleteItemAsync <DeviceInformationItem>(
partitionKey : PartitionKey.NonePartitionKeyValue,
id : item.Id);
}
}
}
}
public async Task ConflictsFeedSetsPartitionKeyRangeIdentity()
{
CosmosContainerCore container = CosmosConflictTests.GetMockedContainer((request, cancellationToken) => {
Assert.IsNotNull(request.DocumentServiceRequest.PartitionKeyRangeIdentity);
return(TestHandler.ReturnSuccess());
});
FeedIterator iterator = container.GetConflicts().GetConflictsStreamIterator();
while (iterator.HasMoreResults)
{
CosmosResponseMessage responseMessage = await iterator.FetchNextSetAsync();
}
}
/// <summary>
/// List the container within a database by calling the GetContainerIterator (scan) API.
/// </summary>
/// <returns></returns>
private static async Task ListContainersInDatabase()
{
Console.WriteLine("\n5. Reading all CosmosContainer resources for a database");
FeedIterator <CosmosContainerSettings> resultSetIterator = database.GetContainersIterator();
while (resultSetIterator.HasMoreResults)
{
foreach (CosmosContainerSettings container in await resultSetIterator.FetchNextSetAsync())
{
Console.WriteLine(container.Id);
}
}
}
public async Task ValidateUserDefinedFunctionsTest()
{
// Prevent failures if previous test did not clean up correctly
await this.scripts.DeleteUserDefinedFunctionAsync("calculateTax");
ToDoActivity item = new ToDoActivity()
{
id = Guid.NewGuid().ToString(),
cost = 9001,
description = "udf_test_item",
status = "Done",
taskNum = 1
};
await this.container.CreateItemAsync <ToDoActivity>(item.status, item);
CosmosUserDefinedFunctionSettings cosmosUserDefinedFunction = await this.scripts.CreateUserDefinedFunctionAsync(
new CosmosUserDefinedFunctionSettings
{
Id = "calculateTax",
Body = @"function(amt) { return amt * 0.05; }"
});
CosmosSqlQueryDefinition sqlQuery = new CosmosSqlQueryDefinition(
"SELECT t.id, t.status, t.cost, udf.calculateTax(t.cost) as total FROM toDoActivity t where t.cost > @expensive and t.status = @status")
.UseParameter("@expensive", 9000)
.UseParameter("@status", "Done");
FeedIterator <dynamic> feedIterator = this.container.CreateItemQuery <dynamic>(
sqlQueryDefinition: sqlQuery,
partitionKey: "Done");
HashSet <string> iterIds = new HashSet <string>();
while (feedIterator.HasMoreResults)
{
foreach (var response in await feedIterator.FetchNextSetAsync())
{
Assert.IsTrue(response.cost > 9000);
Assert.AreEqual(response.cost * .05, response.total);
iterIds.Add(response.id.Value);
}
}
Assert.IsTrue(iterIds.Count > 0);
Assert.IsTrue(iterIds.Contains(item.id));
// Delete existing user defined functions.
await this.scripts.DeleteUserDefinedFunctionAsync(cosmosUserDefinedFunction.Id);
}
public async Task StandByFeedIterator_NoFetchNext()
{
var pkRanges = await this.Container.ClientContext.DocumentClient.ReadPartitionKeyRangeFeedAsync(this.Container.LinkUri);
int expected = 25;
int iterations = 0;
await this.CreateRandomItems(expected, randomPartitionKey : true);
CosmosContainerCore itemsCore = (CosmosContainerCore)this.Container;
string continuationToken = null;
int count = 0;
while (true)
{
ChangeFeedRequestOptions requestOptions = new ChangeFeedRequestOptions()
{
StartTime = DateTime.MinValue
};
FeedIterator feedIterator = itemsCore.GetStandByFeedIterator(continuationToken, requestOptions: requestOptions);
using (CosmosResponseMessage responseMessage =
await feedIterator.FetchNextSetAsync(this.cancellationToken))
{
continuationToken = responseMessage.Headers.Continuation;
if (responseMessage.IsSuccessStatusCode)
{
Collection <ToDoActivity> response = new CosmosJsonSerializerCore().FromStream <CosmosFeedResponseUtil <ToDoActivity> >(responseMessage.Content).Data;
count += response.Count;
}
}
if (count > expected)
{
Assert.Fail($"{count} does not equal {expected}");
}
if (count.Equals(expected))
{
break;
}
if (iterations++ > pkRanges.Count)
{
Assert.Fail("Feed does not contain all elements even after looping through PK ranges. Either the continuation is not moving forward or there is some state problem.");
}
}
}
/// <summary>
/// Run basic database meta data operations as a console application.
/// </summary>
/// <returns></returns>
private static async Task RunDatabaseDemo(CosmosClient client)
{
// An object containing relevant information about the response
DatabaseResponse databaseResponse = await client.CreateDatabaseIfNotExistsAsync(databaseId, 10000);
// A client side reference object that allows additional operations like ReadAsync
CosmosDatabase database = databaseResponse;
// The response from Azure Cosmos
CosmosDatabaseSettings settings = databaseResponse;
Console.WriteLine($"\n1. Create a database resource with id: {settings.Id} and last modified time stamp: {settings.LastModified}");
Console.WriteLine($"\n2. Create a database resource request charge: {databaseResponse.RequestCharge} and Activity Id: {databaseResponse.ActivityId}");
// Read the database from Azure Cosmos
DatabaseResponse readResponse = await database.ReadAsync();
Console.WriteLine($"\n3. Read a database: {readResponse.Resource.Id}");
await readResponse.Database.CreateContainerAsync("testContainer", "/pk");
// Get the current throughput for the database
int?throughput = await database.ReadProvisionedThroughputAsync();
if (throughput.HasValue)
{
Console.WriteLine($"\n4. Read a database throughput: {throughput.Value}");
// Update the current throughput for the database
await database.ReplaceProvisionedThroughputAsync(11000);
}
Console.WriteLine("\n5. Reading all databases resources for an account");
FeedIterator <CosmosDatabaseSettings> iterator = client.GetDatabasesIterator();
do
{
foreach (CosmosDatabaseSettings db in await iterator.FetchNextSetAsync())
{
Console.WriteLine(db.Id);
}
} while (iterator.HasMoreResults);
// Delete the database from Azure Cosmos.
await database.DeleteAsync();
Console.WriteLine($"\n6. Database {database.Id} deleted.");
}
public async Task StreamIteratorTest()
{
string containerName = Guid.NewGuid().ToString();
string partitionKeyPath = "/users";
ContainerResponse containerResponse = await this.cosmosDatabase.CreateContainerAsync(containerName, partitionKeyPath);
Assert.AreEqual(HttpStatusCode.Created, containerResponse.StatusCode);
Assert.AreEqual(containerName, containerResponse.Resource.Id);
Assert.AreEqual(partitionKeyPath, containerResponse.Resource.PartitionKey.Paths.First());
containerName = Guid.NewGuid().ToString();
containerResponse = await this.cosmosDatabase.CreateContainerAsync(containerName, partitionKeyPath);
Assert.AreEqual(HttpStatusCode.Created, containerResponse.StatusCode);
Assert.AreEqual(containerName, containerResponse.Resource.Id);
Assert.AreEqual(partitionKeyPath, containerResponse.Resource.PartitionKey.Paths.First());
HashSet <string> containerIds = new HashSet <string>();
FeedIterator resultSet = this.cosmosDatabase.GetContainersStreamIterator(
maxItemCount: 1,
requestOptions: new QueryRequestOptions());
while (resultSet.HasMoreResults)
{
using (CosmosResponseMessage message = await resultSet.FetchNextSetAsync())
{
Assert.AreEqual(HttpStatusCode.OK, message.StatusCode);
CosmosJsonSerializerCore defaultJsonSerializer = new CosmosJsonSerializerCore();
dynamic containers = defaultJsonSerializer.FromStream <dynamic>(message.Content).DocumentCollections;
foreach (dynamic container in containers)
{
string id = container.id.ToString();
containerIds.Add(id);
}
}
}
Assert.IsTrue(containerIds.Count > 0, "The iterator did not find any containers.");
Assert.IsTrue(containerIds.Contains(containerName), "The iterator did not find the created container");
containerResponse = await containerResponse.Container.DeleteAsync();
Assert.AreEqual(HttpStatusCode.NoContent, containerResponse.StatusCode);
}
public async Task CosmosConflictsStreamIteratorBuildsSettings()
{
string conflictResponsePayload = @"{ 'Data':[{
id: 'Conflict1',
operationType: 'Replace',
resourceType: 'trigger'
}]}";
CosmosClient mockClient = MockCosmosUtil.CreateMockCosmosClient(
(cosmosClientBuilder) => cosmosClientBuilder.WithConnectionModeDirect());
CosmosContainer container = mockClient.GetContainer("database", "container");
FeedIterator feedIterator = container.GetConflicts().GetConflictsStreamIterator();
TestHandler testHandler = new TestHandler((request, cancellationToken) =>
{
Assert.AreEqual(OperationType.ReadFeed, request.OperationType);
Assert.AreEqual(ResourceType.Conflict, request.ResourceType);
CosmosResponseMessage handlerResponse = TestHandler.ReturnSuccess().Result;
MemoryStream stream = new MemoryStream();
StreamWriter writer = new StreamWriter(stream);
writer.Write(conflictResponsePayload);
writer.Flush();
stream.Position = 0;
handlerResponse.Content = stream;
return(Task.FromResult(handlerResponse));
});
mockClient.RequestHandler.InnerHandler = testHandler;
CosmosResponseMessage streamResponse = await feedIterator.FetchNextSetAsync();
Collection <CosmosConflictSettings> response = new CosmosJsonSerializerCore().FromStream <CosmosFeedResponseUtil <CosmosConflictSettings> >(streamResponse.Content).Data;
Assert.AreEqual(1, response.Count());
CosmosConflictSettings responseSettings = response.FirstOrDefault();
Assert.IsNotNull(responseSettings);
Assert.AreEqual("Conflict1", responseSettings.Id);
Assert.AreEqual(Cosmos.OperationKind.Replace, responseSettings.OperationKind);
Assert.AreEqual(typeof(CosmosTriggerSettings), responseSettings.ResourceType);
}
private static async Task QueryWithSqlParameters(CosmosContainer container)
{
// Query using two properties within each item. WHERE Id == "" AND Address.City == ""
// notice here how we are doing an equality comparison on the string value of City
CosmosSqlQueryDefinition query = new CosmosSqlQueryDefinition("SELECT * FROM Families f WHERE f.id = @id AND f.Address.City = @city")
.UseParameter("@id", "AndersonFamily")
.UseParameter("@city", "Seattle");
List <Family> results = new List <Family>();
FeedIterator <Family> resultSetIterator = container.CreateItemQuery <Family>(query, partitionKey: new PartitionKey("Anderson"));
while (resultSetIterator.HasMoreResults)
{
results.AddRange((await resultSetIterator.FetchNextSetAsync()));
}
Assert("Expected only 1 family", results.Count == 1);
}
private static async Task ItemFeed(CosmosContainer container)
{
List <Family> families = new List <Family>();
// SQL
FeedIterator <Family> setIterator = container.GetItemsIterator <Family>(maxItemCount: 1);
while (setIterator.HasMoreResults)
{
int count = 0;
foreach (Family item in await setIterator.FetchNextSetAsync())
{
Assert("Should only return 1 result at a time.", count <= 1);
families.Add(item);
}
}
Assert("Expected two families", families.ToList().Count == 2);
}
public async Task TriggersIteratorTest()
{
CosmosTriggerSettings cosmosTrigger = await CreateRandomTrigger();
HashSet <string> settings = new HashSet <string>();
FeedIterator <CosmosTriggerSettings> iter = this.scripts.GetTriggersIterator();;
while (iter.HasMoreResults)
{
foreach (CosmosTriggerSettings storedProcedureSettingsEntry in await iter.FetchNextSetAsync())
{
settings.Add(storedProcedureSettingsEntry.Id);
}
}
Assert.IsTrue(settings.Contains(cosmosTrigger.Id), "The iterator did not return the user defined function definition.");
// Delete existing user defined functions.
await this.scripts.DeleteTriggerAsync(cosmosTrigger.Id);
}
public async Task DatabaseIterator()
{
List <CosmosDatabase> deleteList = new List <CosmosDatabase>();
HashSet <string> databaseIds = new HashSet <string>();
try
{
for (int i = 0; i < 3; i++)
{
DatabaseResponse createResponse = await this.CreateDatabaseHelper();
deleteList.Add(createResponse.Database);
databaseIds.Add(createResponse.Resource.Id);
}
FeedIterator <CosmosDatabaseSettings> feedIterator =
this.cosmosClient.GetDatabasesIterator();
while (feedIterator.HasMoreResults)
{
FeedResponse <CosmosDatabaseSettings> iterator =
await feedIterator.FetchNextSetAsync(this.cancellationToken);
foreach (CosmosDatabaseSettings databaseSettings in iterator)
{
if (databaseIds.Contains(databaseSettings.Id))
{
databaseIds.Remove(databaseSettings.Id);
}
}
}
}
finally
{
foreach (CosmosDatabase database in deleteList)
{
await database.DeleteAsync(cancellationToken : this.cancellationToken);
}
}
Assert.AreEqual(0, databaseIds.Count);
}
private async Task <long> GetRemainingWorkAsync(DocumentServiceLease existingLease, CancellationToken cancellationToken)
{
// Current lease schema maps Token to PKRangeId
string partitionKeyRangeId = existingLease.CurrentLeaseToken;
FeedIterator iterator = this.feedCreator(
partitionKeyRangeId,
existingLease.ContinuationToken,
string.IsNullOrEmpty(existingLease.ContinuationToken));
try
{
CosmosResponseMessage response = await iterator.FetchNextSetAsync(cancellationToken).ConfigureAwait(false);
if (response.StatusCode != HttpStatusCode.NotModified)
{
response.EnsureSuccessStatusCode();
}
long parsedLSNFromSessionToken = RemainingWorkEstimatorCore.TryConvertToNumber(ExtractLsnFromSessionToken(response.Headers[HttpConstants.HttpHeaders.SessionToken]));
Collection <JObject> items = RemainingWorkEstimatorCore.GetItemsFromResponse(response);
long lastQueryLSN = items.Count > 0
? RemainingWorkEstimatorCore.TryConvertToNumber(RemainingWorkEstimatorCore.GetFirstItemLSN(items)) - 1
: parsedLSNFromSessionToken;
if (lastQueryLSN < 0)
{
return(1);
}
long leaseTokenRemainingWork = parsedLSNFromSessionToken - lastQueryLSN;
return(leaseTokenRemainingWork < 0 ? 0 : leaseTokenRemainingWork);
}
catch (Exception clientException)
{
DefaultTrace.TraceException(clientException);
DefaultTrace.TraceWarning("GetEstimateWork > exception: lease token '{0}'", existingLease.CurrentLeaseToken);
throw;
}
}
public async Task StandByFeedIterator_WithInexistentRange()
{
// Add some random range, this will force the failure
List <CompositeContinuationToken> corruptedTokens = new List <CompositeContinuationToken>
{
new CompositeContinuationToken()
{
Range = new Documents.Routing.Range <string>("whatever", "random", true, false),
Token = "oops"
}
};
string corruptedTokenSerialized = JsonConvert.SerializeObject(corruptedTokens);
CosmosContainerCore itemsCore = (CosmosContainerCore)this.Container;
FeedIterator setIteratorNew =
itemsCore.GetStandByFeedIterator(corruptedTokenSerialized);
CosmosResponseMessage responseMessage =
await setIteratorNew.FetchNextSetAsync(this.cancellationToken);
Assert.Fail("Should have thrown.");
}
public async Task IteratorTest()
{
string sprocBody = "function() { { var x = 42; } }";
int numberOfSprocs = 3;
string[] sprocIds = new string[numberOfSprocs];
for (int i = 0; i < numberOfSprocs; i++)
{
string sprocId = Guid.NewGuid().ToString();
sprocIds[i] = sprocId;
StoredProcedureResponse storedProcedureResponse =
await this.scripts.CreateStoredProcedureAsync(new CosmosStoredProcedureProperties(sprocId, sprocBody));
Assert.AreEqual(HttpStatusCode.Created, storedProcedureResponse.StatusCode);
}
List <string> readSprocIds = new List <string>();
FeedIterator <CosmosStoredProcedureProperties> iter = this.scripts.GetStoredProceduresIterator();
while (iter.HasMoreResults)
{
FeedResponse <CosmosStoredProcedureProperties> currentResultSet = await iter.FetchNextSetAsync();
{
foreach (CosmosStoredProcedureProperties storedProcedureSettingsEntry in currentResultSet)
{
readSprocIds.Add(storedProcedureSettingsEntry.Id);
}
}
}
CollectionAssert.AreEquivalent(sprocIds, readSprocIds);
}
public async Task CrossPartitionBiDirectionalItemReadFeedTest(bool useStatelessIteration)
{
//create items
const int total = 30;
const int maxItemCount = 10;
List <string> items = new List <string>();
for (int i = 0; i < total; i++)
{
string item = $@"
{{
""id"": ""{i}""
}}";
using (CosmosResponseMessage createResponse = await this.Container.CreateItemStreamAsync(
new Cosmos.PartitionKey(i.ToString()),
CosmosReadFeedTests.GenerateStreamFromString(item)))
{
Assert.IsTrue(createResponse.IsSuccessStatusCode);
}
}
string lastKnownContinuationToken = null;
FeedIterator iter = this.Container.Database.GetContainer(this.Container.Id)
.GetItemsStreamIterator(maxItemCount, continuationToken: lastKnownContinuationToken);
int count = 0;
List <string> forwardOrder = new List <string>();
while (iter.HasMoreResults)
{
if (useStatelessIteration)
{
iter = this.Container.Database.GetContainer(this.Container.Id)
.GetItemsStreamIterator(maxItemCount, continuationToken: lastKnownContinuationToken);
}
using (CosmosResponseMessage response = await iter.FetchNextSetAsync())
{
lastKnownContinuationToken = response.Headers.Continuation;
Assert.IsNotNull(response);
using (StreamReader reader = new StreamReader(response.Content))
{
string json = await reader.ReadToEndAsync();
JArray documents = (JArray)JObject.Parse(json).SelectToken("Documents");
count += documents.Count;
if (documents.Any())
{
forwardOrder.Add(documents.First().SelectToken("id").ToString());
}
}
}
}
Assert.IsNull(lastKnownContinuationToken);
Assert.IsNotNull(forwardOrder);
Assert.AreEqual(total, count);
Assert.IsFalse(forwardOrder.Where(x => string.IsNullOrEmpty(x)).Any());
ItemRequestOptions requestOptions = new ItemRequestOptions();
requestOptions.Properties = requestOptions.Properties = new Dictionary <string, object>();
requestOptions.Properties.Add(HttpConstants.HttpHeaders.EnumerationDirection, (byte)BinaryScanDirection.Reverse);
count = 0;
List <string> reverseOrder = new List <string>();
lastKnownContinuationToken = null;
iter = this.Container.Database.GetContainer(this.Container.Id)
.GetItemsStreamIterator(maxItemCount, continuationToken: lastKnownContinuationToken, requestOptions: requestOptions);
while (iter.HasMoreResults)
{
if (useStatelessIteration)
{
iter = this.Container.Database.GetContainer(this.Container.Id)
.GetItemsStreamIterator(maxItemCount, continuationToken: lastKnownContinuationToken, requestOptions: requestOptions);
}
using (CosmosResponseMessage response = await iter.FetchNextSetAsync())
{
lastKnownContinuationToken = response.Headers.Continuation;
Assert.IsNotNull(response);
using (StreamReader reader = new StreamReader(response.Content))
{
string json = await reader.ReadToEndAsync();
JArray documents = (JArray)JObject.Parse(json).SelectToken("Documents");
count += documents.Count;
if (documents.Any())
{
reverseOrder.Add(documents.First().SelectToken("id").ToString());
}
}
}
}
Assert.IsNull(lastKnownContinuationToken);
Assert.IsNotNull(reverseOrder);
Assert.AreEqual(total, count);
forwardOrder.Reverse();
CollectionAssert.AreEqual(forwardOrder, reverseOrder);
Assert.IsFalse(reverseOrder.Where(x => string.IsNullOrEmpty(x)).Any());
}
public async Task TestJsonSerializerSettings(bool useGateway)
{
CosmosClient cosmosClient = TestCommon.CreateCosmosClient((cosmosClientBuilder) => {
if (useGateway)
{
cosmosClientBuilder.WithCustomJsonSerializer(new CustomJsonSerializer(CustomSerializationTests.GetSerializerWithCustomConverterAndBinder())).WithConnectionModeGateway();
}
else
{
cosmosClientBuilder.WithCustomJsonSerializer(new CustomJsonSerializer(CustomSerializationTests.GetSerializerWithCustomConverterAndBinder())).WithConnectionModeDirect();
}
});
CosmosContainer container = cosmosClient.Databases[databaseName].Containers[partitionedCollectionName];
var rnd = new Random();
var bytes = new byte[100];
rnd.NextBytes(bytes);
var testDocument = new TestDocument(new KerberosTicketHashKey(bytes));
//create and read
ItemResponse <TestDocument> createResponse = await container.CreateItemAsync <TestDocument>(testDocument.Name, testDocument);
ItemResponse <TestDocument> readResponse = await container.ReadItemAsync <TestDocument>(testDocument.Name, testDocument.Id);
AssertEqual(testDocument, readResponse.Resource);
AssertEqual(testDocument, createResponse.Resource);
// upsert
ItemResponse <TestDocument> upsertResponse = await container.UpsertItemAsync <TestDocument>(testDocument.Name, testDocument);
readResponse = await container.ReadItemAsync <TestDocument>(testDocument.Name, testDocument.Id);
AssertEqual(testDocument, readResponse.Resource);
AssertEqual(testDocument, upsertResponse.Resource);
// replace
ItemResponse <TestDocument> replacedResponse = await container.ReplaceItemAsync <TestDocument>(testDocument.Name, testDocument.Id, testDocument);
readResponse = await container.ReadItemAsync <TestDocument>(testDocument.Name, testDocument.Id);
AssertEqual(testDocument, readResponse.Resource);
AssertEqual(testDocument, replacedResponse.Resource);
CosmosSqlQueryDefinition sql = new CosmosSqlQueryDefinition("select * from r");
FeedIterator <TestDocument> feedIterator =
container.CreateItemQuery <TestDocument>(sqlQueryDefinition: sql, partitionKey: testDocument.Name, maxItemCount: 1);
FeedResponse <TestDocument> queryResponse = await feedIterator.FetchNextSetAsync();
AssertEqual(testDocument, queryResponse.First());
//Will add LINQ test once it is available with new V3 OM
// // LINQ Lambda
// var query1 = client.CreateDocumentQuery<TestDocument>(partitionedCollectionUri, options)
// .Where(_ => _.Id.CompareTo(String.Empty) > 0)
// .Select(_ => _.Id);
// string query1Str = query1.ToString();
// var result = query1.ToList();
// Assert.AreEqual(1, result.Count);
// Assert.AreEqual(testDocument.Id, result[0]);
// // LINQ Query
// var query2 =
// from f in client.CreateDocumentQuery<TestDocument>(partitionedCollectionUri, options)
// where f.Id.CompareTo(String.Empty) > 0
// select f.Id;
// string query2Str = query2.ToString();
// var result2 = query2.ToList();
// Assert.AreEqual(1, result2.Count);
// Assert.AreEqual(testDocument.Id, result2[0]);
}
private static async Task QueryPartitionedContainerInParallelAsync(CosmosContainer container)
{
List <Family> familiesSerial = new List <Family>();
string queryText = "SELECT * FROM Families";
// 0 maximum parallel tasks, effectively serial execution
QueryRequestOptions options = new QueryRequestOptions()
{
MaxBufferedItemCount = 100
};
FeedIterator <Family> query = container.CreateItemQuery <Family>(
queryText,
maxConcurrency: 0,
requestOptions: options);
while (query.HasMoreResults)
{
foreach (Family family in await query.FetchNextSetAsync())
{
familiesSerial.Add(family);
}
}
Assert("Parallel Query expected two families", familiesSerial.ToList().Count == 2);
// 1 maximum parallel tasks, 1 dedicated asynchronous task to continuously make REST calls
List <Family> familiesParallel1 = new List <Family>();
query = container.CreateItemQuery <Family>(
queryText,
maxConcurrency: 1,
requestOptions: options);
while (query.HasMoreResults)
{
foreach (Family family in await query.FetchNextSetAsync())
{
familiesParallel1.Add(family);
}
}
Assert("Parallel Query expected two families", familiesParallel1.ToList().Count == 2);
AssertSequenceEqual("Parallel query returns result out of order compared to serial execution", familiesSerial, familiesParallel1);
// 10 maximum parallel tasks, a maximum of 10 dedicated asynchronous tasks to continuously make REST calls
List <Family> familiesParallel10 = new List <Family>();
query = container.CreateItemQuery <Family>(
queryText,
maxConcurrency: 10,
requestOptions: options);
while (query.HasMoreResults)
{
foreach (Family family in await query.FetchNextSetAsync())
{
familiesParallel10.Add(family);
}
}
Assert("Parallel Query expected two families", familiesParallel10.ToList().Count == 2);
AssertSequenceEqual("Parallel query returns result out of order compared to serial execution", familiesSerial, familiesParallel10);
}
public async Task StandByFeedIterator()
{
int totalCount = 0;
string lastcontinuation = string.Empty;
int firstRunTotal = 25;
int batchSize = 25;
Documents.Routing.Range <string> previousRange = null;
Documents.Routing.Range <string> currentRange = null;
int pkRangesCount = (await this.Container.ClientContext.DocumentClient.ReadPartitionKeyRangeFeedAsync(this.Container.LinkUri)).Count;
int visitedPkRanges = 0;
await this.CreateRandomItems(batchSize, randomPartitionKey : true);
CosmosContainerCore itemsCore = (CosmosContainerCore)this.Container;
FeedIterator feedIterator = itemsCore.GetStandByFeedIterator(requestOptions: new ChangeFeedRequestOptions()
{
StartTime = DateTime.MinValue
});
while (feedIterator.HasMoreResults)
{
using (CosmosResponseMessage responseMessage =
await feedIterator.FetchNextSetAsync(this.cancellationToken))
{
lastcontinuation = responseMessage.Headers.Continuation;
List <CompositeContinuationToken> deserializedToken = JsonConvert.DeserializeObject <List <CompositeContinuationToken> >(lastcontinuation);
currentRange = deserializedToken[0].Range;
Assert.AreEqual(pkRangesCount, deserializedToken.Count);
if (responseMessage.IsSuccessStatusCode)
{
Collection <ToDoActivity> response = new CosmosJsonSerializerCore().FromStream <CosmosFeedResponseUtil <ToDoActivity> >(responseMessage.Content).Data;
totalCount += response.Count;
}
if (!currentRange.Equals(previousRange))
{
visitedPkRanges++;
}
if (visitedPkRanges == pkRangesCount && responseMessage.StatusCode == System.Net.HttpStatusCode.NotModified)
{
break;
}
previousRange = currentRange;
}
}
Assert.AreEqual(firstRunTotal, totalCount);
int expectedFinalCount = 50;
previousRange = null;
currentRange = null;
visitedPkRanges = 0;
// Insert another batch of 25 and use the last continuation token from the first cycle
await this.CreateRandomItems(batchSize, randomPartitionKey : true);
FeedIterator setIteratorNew =
itemsCore.GetStandByFeedIterator(lastcontinuation);
while (setIteratorNew.HasMoreResults)
{
using (CosmosResponseMessage responseMessage =
await setIteratorNew.FetchNextSetAsync(this.cancellationToken))
{
lastcontinuation = responseMessage.Headers.Continuation;
currentRange = JsonConvert.DeserializeObject <List <CompositeContinuationToken> >(lastcontinuation)[0].Range;
if (responseMessage.IsSuccessStatusCode)
{
Collection <ToDoActivity> response = new CosmosJsonSerializerCore().FromStream <CosmosFeedResponseUtil <ToDoActivity> >(responseMessage.Content).Data;
totalCount += response.Count;
}
if (!currentRange.Equals(previousRange))
{
visitedPkRanges++;
}
if (visitedPkRanges == pkRangesCount && responseMessage.StatusCode == System.Net.HttpStatusCode.NotModified)
{
break;
}
previousRange = currentRange;
}
}
Assert.AreEqual(expectedFinalCount, totalCount);
}
private static async Task QueryItems()
{
//******************************************************************************************************************
// 1.4 - Query for items by a property other than Id
//
// NOTE: Operations like AsEnumerable(), ToList(), ToArray() will make as many trips to the database
// as required to fetch the entire result-set. Even if you set MaxItemCount to a smaller number.
// MaxItemCount just controls how many results to fetch each trip.
//******************************************************************************************************************
Console.WriteLine("\n1.4 - Querying for a item using its AccountNumber property");
CosmosSqlQueryDefinition query = new CosmosSqlQueryDefinition(
"select * from sales s where s.AccountNumber = @AccountInput ")
.UseParameter("@AccountInput", "Account1");
FeedIterator <SalesOrder> resultSet = container.CreateItemQuery <SalesOrder>(
query,
partitionKey: new PartitionKey("Account1"),
maxItemCount: 1);
List <SalesOrder> allSalesForAccount1 = new List <SalesOrder>();
while (resultSet.HasMoreResults)
{
SalesOrder sale = (await resultSet.FetchNextSetAsync()).First();
Console.WriteLine($"\n1.4.1 Account Number: {sale.AccountNumber}; Id: {sale.Id} ");
allSalesForAccount1.Add(sale);
}
Console.WriteLine($"\n1.4.2 Query found {allSalesForAccount1.Count} items.");
// Use the same query as before but get the cosmos response message to access the stream directly
FeedIterator streamResultSet = container.CreateItemQueryStream(
query,
maxConcurrency: 1,
partitionKey: new PartitionKey("Account1"),
maxItemCount: 10);
List <SalesOrder> allSalesForAccount1FromStream = new List <SalesOrder>();
while (streamResultSet.HasMoreResults)
{
using (CosmosResponseMessage responseMessage = await streamResultSet.FetchNextSetAsync())
{
// Item stream operations do not throw exceptions for better performance
if (responseMessage.IsSuccessStatusCode)
{
dynamic streamResponse = FromStream <dynamic>(responseMessage.Content);
List <SalesOrder> salesOrders = streamResponse.Documents.ToObject <List <SalesOrder> >();
Console.WriteLine($"\n1.4.3 - Item Query via stream {salesOrders.Count}");
allSalesForAccount1FromStream.AddRange(salesOrders);
}
else
{
Console.WriteLine($"Query item from stream failed. Status code: {responseMessage.StatusCode} Message: {responseMessage.ErrorMessage}");
}
}
}
Console.WriteLine($"\n1.4.4 Query found {allSalesForAccount1FromStream.Count} items.");
if (allSalesForAccount1.Count != allSalesForAccount1FromStream.Count)
{
throw new InvalidDataException($"Both query operations should return the same list");
}
}
/// <summary>
/// Import many documents using stored procedure.
/// </summary>
private static async Task RunBulkImport(CosmosContainer container)
{
string inputDirectory = @".\Data\";
string inputFileMask = "*.json";
int maxFiles = 2000;
int maxScriptSize = 50000;
// 1. Get the files.
string[] fileNames = Directory.GetFiles(inputDirectory, inputFileMask);
DirectoryInfo di = new DirectoryInfo(inputDirectory);
FileInfo[] fileInfos = di.GetFiles(inputFileMask);
// 2. Prepare for import.
int currentCount = 0;
int fileCount = maxFiles != 0 ? Math.Min(maxFiles, fileNames.Length) : fileNames.Length;
// 3. Create stored procedure for this script.
string scriptId = "BulkImport";
string body = File.ReadAllText(@".\JS\BulkImport.js");
await TryDeleteStoredProcedure(container, scriptId);
CosmosScripts cosmosScripts = container.GetScripts();
StoredProcedureResponse sproc = await cosmosScripts.CreateStoredProcedureAsync(new CosmosStoredProcedureSettings(scriptId, body));
// 4. Create a batch of docs (MAX is limited by request size (2M) and to script for execution.
// We send batches of documents to create to script.
// Each batch size is determined by MaxScriptSize.
// MaxScriptSize should be so that:
// -- it fits into one request (MAX request size is 16Kb).
// -- it doesn't cause the script to time out.
// -- it is possible to experiment with MaxScriptSize to get best performance given number of throttles, etc.
while (currentCount < fileCount)
{
// 5. Create args for current batch.
// Note that we could send a string with serialized JSON and JSON.parse it on the script side,
// but that would cause script to run longer. Since script has timeout, unload the script as much
// as we can and do the parsing by client and framework. The script will get JavaScript objects.
string argsJson = CreateBulkInsertScriptArguments(fileNames, currentCount, fileCount, maxScriptSize);
dynamic[] args = new dynamic[] { JsonConvert.DeserializeObject <dynamic>(argsJson) };
// 6. execute the batch.
StoredProcedureExecuteResponse <int> scriptResult = await cosmosScripts.ExecuteStoredProcedureAsync <dynamic, int>(new PartitionKey("Andersen"), scriptId, args);
// 7. Prepare for next batch.
int currentlyInserted = scriptResult.Resource;
currentCount += currentlyInserted;
}
// 8. Validate
int numDocs = 0;
FeedIterator <dynamic> setIterator = container.GetItemsIterator <dynamic>();
while (setIterator.HasMoreResults)
{
FeedResponse <dynamic> response = await setIterator.FetchNextSetAsync();
numDocs += response.Count();
}
Console.WriteLine("Found {0} documents in the collection. There were originally {1} files in the Data directory\r\n", numDocs, fileCount);
}