public void ValidateTransformQuery()
{
DocumentClient client = TestCommon.CreateClient(true);
IQueryable <dynamic> dbQuery = client.CreateDatabaseQuery(@"select * from root r where r.id=""db123""").AsQueryable();
foreach (CosmosDatabaseSettings db in dbQuery)
{
TestCommon.Delete <CosmosDatabaseSettings>(client, db.ResourceId);
}
CosmosDatabaseSettings database = client.Create <CosmosDatabaseSettings>(null,
new CosmosDatabaseSettings {
Id = "db123"
});
dbQuery = client.CreateDatabaseQuery(@"select * from root r where r.id=""db123""").AsQueryable();
foreach (CosmosDatabaseSettings db in dbQuery)
{
Assert.AreEqual(db.Id, "db123");
}
Assert.AreNotEqual(0, System.Linq.Dynamic.Core.DynamicQueryableExtensions.AsEnumerable(dbQuery).Count());
IQueryable <dynamic> dbIdQuery = client.CreateDatabaseQuery(@"select r._rid from root r where r.id=""db123""").AsQueryable();
Assert.AreNotEqual(0, System.Linq.Dynamic.Core.DynamicQueryableExtensions.AsEnumerable(dbIdQuery).Count());
CosmosContainerSettings collection = new CosmosContainerSettings
{
Id = Guid.NewGuid().ToString("N")
};
collection.IndexingPolicy.IndexingMode = IndexingMode.Consistent;
collection = client.Create <CosmosContainerSettings>(database.ResourceId, collection);
int documentsToCreate = 100;
for (int i = 0; i < documentsToCreate; i++)
{
dynamic myDocument = new Document();
myDocument.Id = "doc" + i;
myDocument.Title = "MyBook"; //Simple Property.
myDocument.Languages = new Language[] { new Language {
Name = "English", Copyright = "London Publication"
}, new Language {
Name = "French", Copyright = "Paris Publication"
} }; //Array Property
myDocument.Author = new Author {
Name = "Don", Location = "France"
}; //Complex Property
myDocument.Price = 9.99;
myDocument = client.CreateDocumentAsync(collection.DocumentsLink, myDocument).Result;
}
//Read response as dynamic.
IQueryable <dynamic> docQuery = client.CreateDocumentQuery(collection.DocumentsLink, @"select * from root r where r.Title=""MyBook""", null);
IDocumentQuery <dynamic> DocumentQuery = docQuery.AsDocumentQuery();
FeedResponse <dynamic> queryResponse = DocumentQuery.ExecuteNextAsync().Result;
Assert.IsNotNull(queryResponse.ResponseHeaders, "ResponseHeaders is null");
Assert.IsNotNull(queryResponse.ActivityId, "ActivityId is null");
Assert.AreEqual(documentsToCreate, queryResponse.Count);
foreach (dynamic myBook in queryResponse)
{
Assert.AreEqual(myBook.Title, "MyBook");
}
client.DeleteDocumentCollectionAsync(collection.SelfLink).Wait();
}
/// <inheritdoc/>
public async Task <List <Instance> > GetInstancesInStateOfInstanceOwner(int instanceOwnerPartyId, string instanceState)
{
List <Instance> instances = new List <Instance>();
string instanceOwnerPartyIdString = instanceOwnerPartyId.ToString();
FeedOptions feedOptions = new FeedOptions
{
PartitionKey = new PartitionKey(instanceOwnerPartyIdString)
};
if (_settings.CollectMetrics)
{
feedOptions.PopulateQueryMetrics = true;
}
IQueryable <Instance> filter;
if (instanceState.Equals("active"))
{
filter = _client.CreateDocumentQuery <Instance>(_collectionUri, feedOptions)
.Where(i => i.InstanceOwner.PartyId == instanceOwnerPartyIdString)
.Where(i => (!i.VisibleAfter.HasValue || i.VisibleAfter <= DateTime.UtcNow))
.Where(i => !i.Status.IsSoftDeleted)
.Where(i => !i.Status.IsHardDeleted)
.Where(i => !i.Status.IsArchived);
}
else if (instanceState.Equals("deleted"))
{
filter = _client.CreateDocumentQuery <Instance>(_collectionUri, feedOptions)
.Where(i => i.InstanceOwner.PartyId == instanceOwnerPartyIdString)
.Where(i => i.Status.IsSoftDeleted)
.Where(i => !i.Status.IsHardDeleted);
}
else if (instanceState.Equals("archived"))
{
filter = _client.CreateDocumentQuery <Instance>(_collectionUri, feedOptions)
.Where(i => i.InstanceOwner.PartyId == instanceOwnerPartyIdString)
.Where(i => i.Status.IsArchived)
.Where(i => !i.Status.IsSoftDeleted)
.Where(i => !i.Status.IsHardDeleted)
.OrderByDescending(i => i.Status.Archived);
}
else
{
// empty list
return(instances);
}
IDocumentQuery <Instance> query = filter.AsDocumentQuery();
FeedResponse <Instance> feedResponse = await query.ExecuteNextAsync <Instance>();
if (_settings.CollectMetrics)
{
_logger.LogError($"Metrics retrieving {instanceState} instances for {instanceOwnerPartyId}: {JsonConvert.SerializeObject(feedResponse.QueryMetrics)}");
}
instances = feedResponse.ToList();
await PostProcess(instances);
return(instances);
}
private async Task <List <T> > ToListAsync <T>(QueryStream createStreamQuery, Query <T> createQuery, string queryText)
{
FeedIterator feedStreamIterator = createStreamQuery(queryText, null, RequestOptions);
List <T> streamResults = new List <T>();
while (feedStreamIterator.HasMoreResults)
{
ResponseMessage response = await feedStreamIterator.ReadNextAsync();
response.EnsureSuccessStatusCode();
StreamReader sr = new StreamReader(response.Content);
string result = await sr.ReadToEndAsync();
ICollection <T> responseResults = JsonConvert.DeserializeObject <CosmosFeedResponseUtil <T> >(result).Data;
Assert.IsTrue(responseResults.Count <= 1);
streamResults.AddRange(responseResults);
}
string continuationToken = null;
List <T> pagedStreamResults = new List <T>();
do
{
FeedIterator pagedFeedIterator = createStreamQuery(queryText, continuationToken, RequestOptions);
ResponseMessage response = await pagedFeedIterator.ReadNextAsync();
response.EnsureSuccessStatusCode();
ICollection <T> responseResults = TestCommon.Serializer.FromStream <CosmosFeedResponseUtil <T> >(response.Content).Data;
Assert.IsTrue(responseResults.Count <= 1);
pagedStreamResults.AddRange(responseResults);
continuationToken = response.Headers.ContinuationToken;
} while (continuationToken != null);
Assert.AreEqual(pagedStreamResults.Count, streamResults.Count);
// Both lists should be the same
string streamResultString = JsonConvert.SerializeObject(streamResults);
string streamPagedResultString = JsonConvert.SerializeObject(pagedStreamResults);
Assert.AreEqual(streamPagedResultString, streamResultString);
FeedIterator <T> feedIterator = createQuery(queryText, null, RequestOptions);
List <T> results = new List <T>();
while (feedIterator.HasMoreResults)
{
FeedResponse <T> iterator = await feedIterator.ReadNextAsync();
Assert.IsTrue(iterator.Count <= 1);
Assert.IsTrue(iterator.Resource.Count() <= 1);
results.AddRange(iterator);
}
continuationToken = null;
List <T> pagedResults = new List <T>();
do
{
FeedIterator <T> pagedFeedIterator = createQuery(queryText, continuationToken, RequestOptions);
FeedResponse <T> iterator = await pagedFeedIterator.ReadNextAsync();
Assert.IsTrue(iterator.Count <= 1);
Assert.IsTrue(iterator.Resource.Count() <= 1);
pagedResults.AddRange(iterator);
continuationToken = iterator.ContinuationToken;
} while (continuationToken != null);
Assert.AreEqual(pagedResults.Count, results.Count);
// Both lists should be the same
string resultString = JsonConvert.SerializeObject(results);
string pagedResultString = JsonConvert.SerializeObject(pagedResults);
Assert.AreEqual(pagedResultString, resultString);
Assert.AreEqual(streamPagedResultString, resultString);
return(results);
}
public async Task ChangeFeedIteratorCore_PartitionKey_OfT_ReadAll()
{
int totalCount = 0;
int firstRunTotal = 25;
int batchSize = 25;
string pkToRead = "pkToRead";
string otherPK = "otherPK";
for (int i = 0; i < batchSize; i++)
{
await this.Container.CreateItemAsync(this.CreateRandomToDoActivity(pkToRead));
}
for (int i = 0; i < batchSize; i++)
{
await this.Container.CreateItemAsync(this.CreateRandomToDoActivity(otherPK));
}
ContainerCore itemsCore = this.Container;
FeedIterator <ToDoActivity> feedIterator = itemsCore.GetChangeFeedIterator <ToDoActivity>(new PartitionKey(pkToRead), changeFeedRequestOptions: new ChangeFeedRequestOptions()
{
StartTime = DateTime.MinValue.ToUniversalTime()
});
while (feedIterator.HasMoreResults)
{
FeedResponse <ToDoActivity> feedResponse = await feedIterator.ReadNextAsync(this.cancellationToken);
totalCount += feedResponse.Count;
foreach (ToDoActivity toDoActivity in feedResponse)
{
Assert.AreEqual(pkToRead, toDoActivity.status);
}
}
Assert.AreEqual(firstRunTotal, totalCount);
int expectedFinalCount = 50;
// Insert another batch of 25 and use the last FeedToken from the first cycle
for (int i = 0; i < batchSize; i++)
{
await this.Container.CreateItemAsync(this.CreateRandomToDoActivity(pkToRead));
}
FeedIterator <ToDoActivity> setIteratorNew = itemsCore.GetChangeFeedIterator <ToDoActivity>(feedToken: feedIterator.FeedToken, changeFeedRequestOptions: new ChangeFeedRequestOptions()
{
StartTime = DateTime.MinValue.ToUniversalTime()
});
while (setIteratorNew.HasMoreResults)
{
FeedResponse <ToDoActivity> feedResponse = await setIteratorNew.ReadNextAsync(this.cancellationToken);
totalCount += feedResponse.Count;
foreach (ToDoActivity toDoActivity in feedResponse)
{
Assert.AreEqual(pkToRead, toDoActivity.status);
}
}
Assert.AreEqual(expectedFinalCount, totalCount);
}
public async Task ContainerPartitionResourcePermissionTest(ConnectionMode connectionMode)
{
CosmosClientOptions cosmosClientOptions = new CosmosClientOptions()
{
ConnectionMode = connectionMode
};
CosmosClient cosmosClient = TestCommon.CreateCosmosClient(cosmosClientOptions);
Database database = await cosmosClient.CreateDatabaseIfNotExistsAsync("PermissionTest");
//create user
string userId = Guid.NewGuid().ToString();
UserResponse userResponse = await database.CreateUserAsync(userId);
Assert.AreEqual(HttpStatusCode.Created, userResponse.StatusCode);
Assert.AreEqual(userId, userResponse.Resource.Id);
User user = userResponse.User;
//create resource
string containerId = Guid.NewGuid().ToString();
ContainerResponse containerResponse = await database.CreateContainerAsync(
id : containerId,
partitionKeyPath : "/id");
Assert.AreEqual(HttpStatusCode.Created, containerResponse.StatusCode);
Container container = containerResponse.Container;
// Create items to read
ToDoActivity itemAccess = ToDoActivity.CreateRandomToDoActivity();
ToDoActivity itemNoAccess = ToDoActivity.CreateRandomToDoActivity();
await container.CreateItemAsync <ToDoActivity>(
itemAccess,
new PartitionKey(itemAccess.id));
await container.CreateItemAsync <ToDoActivity>(
itemNoAccess,
new PartitionKey(itemNoAccess.id));
//create permission
string permissionId = Guid.NewGuid().ToString();
PartitionKey partitionKey = new PartitionKey(itemAccess.id);
PermissionProperties permissionProperties = new PermissionProperties(
permissionId,
PermissionMode.Read,
container,
partitionKey);
PermissionResponse permissionResponse = await user.CreatePermissionAsync(permissionProperties);
PermissionProperties permission = permissionResponse.Resource;
Assert.AreEqual(HttpStatusCode.Created, userResponse.StatusCode);
Assert.AreEqual(permissionId, permission.Id);
Assert.AreEqual(permissionProperties.PermissionMode, permission.PermissionMode);
using (CosmosClient tokenCosmosClient = TestCommon.CreateCosmosClient(clientOptions: cosmosClientOptions, resourceToken: permission.Token))
{
Container tokenContainer = tokenCosmosClient.GetContainer(database.Id, containerId);
await tokenContainer.ReadItemAsync <ToDoActivity>(itemAccess.id, new PartitionKey(itemAccess.id));
try
{
await tokenContainer.ReadItemAsync <ToDoActivity>(itemNoAccess.id, new PartitionKey(itemNoAccess.id));
Assert.Fail();
}
catch (CosmosException ex)
{
Assert.AreEqual(HttpStatusCode.Forbidden, ex.StatusCode);
}
QueryRequestOptions queryRequestOptions = new QueryRequestOptions()
{
PartitionKey = new PartitionKey(itemAccess.id)
};
FeedIterator <ToDoActivity> feedIterator = tokenContainer.GetItemQueryIterator <ToDoActivity>(
queryText: "select * from T",
requestOptions: queryRequestOptions);
List <ToDoActivity> result = new List <ToDoActivity>();
while (feedIterator.HasMoreResults)
{
FeedResponse <ToDoActivity> toDoActivities = await feedIterator.ReadNextAsync();
result.AddRange(toDoActivities);
}
Assert.AreEqual(1, result.Count);
// Test query with no service interop via gateway query plan to replicate x32 app
ContainerInternal containerCore = (ContainerInlineCore)tokenContainer;
MockCosmosQueryClient mock = new MockCosmosQueryClient(
clientContext: containerCore.ClientContext,
cosmosContainerCore: containerCore,
forceQueryPlanGatewayElseServiceInterop: true);
Container tokenGatewayQueryPlan = new ContainerInlineCore(
containerCore.ClientContext,
(DatabaseInternal)containerCore.Database,
containerCore.Id,
mock);
FeedIterator <ToDoActivity> feedIteratorGateway = tokenGatewayQueryPlan.GetItemQueryIterator <ToDoActivity>(
queryText: "select * from T",
requestOptions: queryRequestOptions);
List <ToDoActivity> resultGateway = new List <ToDoActivity>();
while (feedIteratorGateway.HasMoreResults)
{
FeedResponse <ToDoActivity> toDoActivities = await feedIteratorGateway.ReadNextAsync();
resultGateway.AddRange(toDoActivities);
}
Assert.AreEqual(1, resultGateway.Count);
}
}
private static async Task RunBulkImport(DocumentClient client, string collectionLink)
{
string inputDirectory = @".\Data\";
string inputFileMask = "*.json";
int maxFiles = 2000;
int maxScriptSize = 50000;
// 1. Get the files.
string[] fileNames = Directory.GetFiles(inputDirectory, inputFileMask);
DirectoryInfo di = new DirectoryInfo(inputDirectory);
FileInfo[] fileInfos = di.GetFiles(inputFileMask);
// 2. Prepare for import.
int currentCount = 0;
int fileCount = maxFiles != 0 ? Math.Min(maxFiles, fileNames.Length) : fileNames.Length;
// 3. Create stored procedure for this script.
string body = File.ReadAllText(@".\JS\BulkImport.js");
StoredProcedure sproc = new StoredProcedure
{
Id = "BulkImport",
Body = body
};
await TryDeleteStoredProcedure(client, collectionLink, sproc.Id);
sproc = await client.CreateStoredProcedureAsync(collectionLink, sproc);
Stopwatch sp = new Stopwatch();
sp.Start();
// 4. Create a batch of docs (MAX is limited by request size (2M) and to script for execution.
// We send batches of documents to create to script.
// Each batch size is determined by MaxScriptSize.
// MaxScriptSize should be so that:
// -- it fits into one request (MAX reqest size is 16Kb).
// -- it doesn't cause the script to time out.
// -- it is possible to experiment with MaxScriptSize to get best perf given number of throttles, etc.
while (currentCount < fileCount)
{
// 5. Create args for current batch.
// Note that we could send a string with serialized JSON and JSON.parse it on the script side,
// but that would cause script to run longer. Since script has timeout, unload the script as much
// as we can and do the parsing by client and framework. The script will get JavaScript objects.
string argsJson = CreateBulkInsertScriptArguments(fileNames, currentCount, fileCount, maxScriptSize);
var args = new dynamic[] { JsonConvert.DeserializeObject <dynamic>(argsJson) };
// 6. execute the batch.
StoredProcedureResponse <int> scriptResult = await client.ExecuteStoredProcedureAsync <int>(
sproc.SelfLink,
args);
// 7. Prepare for next batch.
int currentlyInserted = scriptResult.Response;
currentCount += currentlyInserted;
}
// 8. Validate
int numDocs = 0;
string continuation = string.Empty;
do
{
// Read document feed and count the number of documents.
FeedResponse <dynamic> response = await client.ReadDocumentFeedAsync(collectionLink, new FeedOptions { RequestContinuation = continuation });
numDocs += response.Count;
// Get the continuation so that we know when to stop.
continuation = response.ResponseContinuation;
}while (!string.IsNullOrEmpty(continuation));
Console.WriteLine("Found {0} documents in the collection inserted in {1}ms\r\n", numDocs, sp.Elapsed.Milliseconds);
}
protected override async Task <FeedResponse <CosmosElement> > ExecuteInternalAsync(CancellationToken token)
{
CollectionCache collectionCache = await this.Client.GetCollectionCacheAsync();
PartitionKeyRangeCache partitionKeyRangeCache = await this.Client.GetPartitionKeyRangeCache();
IDocumentClientRetryPolicy retryPolicyInstance = this.Client.ResetSessionTokenRetryPolicy.GetRequestPolicy();
retryPolicyInstance = new InvalidPartitionExceptionRetryPolicy(collectionCache, retryPolicyInstance);
if (base.ResourceTypeEnum.IsPartitioned())
{
retryPolicyInstance = new PartitionKeyRangeGoneRetryPolicy(
collectionCache,
partitionKeyRangeCache,
PathsHelper.GetCollectionPath(base.ResourceLink),
retryPolicyInstance);
}
return(await BackoffRetryUtility <FeedResponse <CosmosElement> > .ExecuteAsync(
async() =>
{
this.fetchExecutionRangeAccumulator.BeginFetchRange();
++this.retries;
FeedResponse <CosmosElement> response = await this.ExecuteOnceAsync(retryPolicyInstance, token);
if (!string.IsNullOrEmpty(response.ResponseHeaders[HttpConstants.HttpHeaders.QueryMetrics]))
{
this.fetchExecutionRangeAccumulator.EndFetchRange(
response.ActivityId,
response.Count,
this.retries);
response = new FeedResponse <CosmosElement>(
response,
response.Count,
response.Headers,
response.UseETagAsContinuation,
new Dictionary <string, QueryMetrics>
{
{
SinglePartitionKeyId,
QueryMetrics.CreateFromDelimitedStringAndClientSideMetrics(
response.ResponseHeaders[HttpConstants.HttpHeaders.QueryMetrics],
new ClientSideMetrics(
this.retries,
response.RequestCharge,
this.fetchExecutionRangeAccumulator.GetExecutionRanges(),
string.IsNullOrEmpty(response.ResponseContinuation) ? new List <Tuple <string, SchedulingTimeSpan> >()
{
new Tuple <string, SchedulingTimeSpan>(SinglePartitionKeyId, this.fetchSchedulingMetrics.Elapsed)
} : new List <Tuple <string, SchedulingTimeSpan> >()))
}
},
response.RequestStatistics,
response.DisallowContinuationTokenMessage,
response.ResponseLengthBytes);
}
this.retries = -1;
return response;
},
retryPolicyInstance,
token));
}
private async Task <List <T> > ToListAsync <T>(
QueryStream createStreamQuery,
Query <T> createQuery,
string queryText,
QueryRequestOptions requestOptions)
{
HttpStatusCode expectedStatus = HttpStatusCode.OK;
FeedIterator feedStreamIterator = createStreamQuery(queryText, null, requestOptions);
List <T> streamResults = new List <T>();
while (feedStreamIterator.HasMoreResults)
{
ResponseMessage response = await feedStreamIterator.ReadNextAsync();
response.EnsureSuccessStatusCode();
Assert.AreEqual(expectedStatus, response.StatusCode);
StreamReader sr = new StreamReader(response.Content);
string result = await sr.ReadToEndAsync();
ICollection <T> responseResults;
responseResults = JsonConvert.DeserializeObject <CosmosFeedResponseUtil <T> >(result).Data;
Assert.IsTrue(responseResults.Count <= 1);
streamResults.AddRange(responseResults);
}
string continuationToken = null;
List <T> pagedStreamResults = new List <T>();
do
{
FeedIterator pagedFeedIterator = createStreamQuery(queryText, continuationToken, requestOptions);
ResponseMessage response = await pagedFeedIterator.ReadNextAsync();
response.EnsureSuccessStatusCode();
Assert.AreEqual(expectedStatus, response.StatusCode);
IEnumerable <T> responseResults = TestCommon.SerializerCore.FromStream <CosmosFeedResponseUtil <T> >(response.Content).Data;
Assert.IsTrue(responseResults.Count() <= 1);
pagedStreamResults.AddRange(responseResults);
continuationToken = response.Headers.ContinuationToken;
Assert.AreEqual(response.ContinuationToken, response.Headers.ContinuationToken);
} while (continuationToken != null);
Assert.AreEqual(pagedStreamResults.Count, streamResults.Count);
// Both lists should be the same if not PermssionsProperties. PermissionProperties will have a different ResouceToken in the payload when read.
string streamResultString = JsonConvert.SerializeObject(streamResults);
string streamPagedResultString = JsonConvert.SerializeObject(pagedStreamResults);
if (typeof(T) != typeof(PermissionProperties))
{
Assert.AreEqual(streamPagedResultString, streamResultString);
}
FeedIterator <T> feedIterator = createQuery(queryText, null, requestOptions);
List <T> results = new List <T>();
while (feedIterator.HasMoreResults)
{
FeedResponse <T> response = await feedIterator.ReadNextAsync();
Assert.AreEqual(expectedStatus, response.StatusCode);
Assert.IsTrue(response.Count <= 1);
Assert.IsTrue(response.Resource.Count() <= 1);
results.AddRange(response);
}
continuationToken = null;
List <T> pagedResults = new List <T>();
do
{
FeedIterator <T> pagedFeedIterator = createQuery(queryText, continuationToken, requestOptions);
FeedResponse <T> response = await pagedFeedIterator.ReadNextAsync();
Assert.AreEqual(expectedStatus, response.StatusCode);
Assert.IsTrue(response.Count <= 1);
Assert.IsTrue(response.Resource.Count() <= 1);
pagedResults.AddRange(response);
continuationToken = response.ContinuationToken;
} while (continuationToken != null);
Assert.AreEqual(pagedResults.Count, results.Count);
// Both lists should be the same
string resultString = JsonConvert.SerializeObject(results);
string pagedResultString = JsonConvert.SerializeObject(pagedResults);
if (typeof(T) != typeof(PermissionProperties))
{
Assert.AreEqual(pagedResultString, resultString);
Assert.AreEqual(streamPagedResultString, resultString);
}
return(results);
}
public async Task ReadFeedIteratorCore_OfT_ReadNextAsync()
{
string continuation = "TBD";
ResponseMessage responseMessage = new ResponseMessage(HttpStatusCode.OK);
responseMessage.Headers.ContinuationToken = continuation;
responseMessage.Headers[Documents.HttpConstants.HttpHeaders.ItemCount] = "1";
responseMessage.Content = new MemoryStream(Encoding.UTF8.GetBytes("{}"));
Mock <CosmosClientContext> cosmosClientContext = new Mock <CosmosClientContext>();
cosmosClientContext.Setup(c => c.ClientOptions).Returns(new CosmosClientOptions());
cosmosClientContext
.Setup(c => c.ProcessResourceOperationStreamAsync(
It.IsAny <string>(),
It.Is <Documents.ResourceType>(rt => rt == Documents.ResourceType.Document),
It.IsAny <Documents.OperationType>(),
It.IsAny <RequestOptions>(),
It.IsAny <ContainerInternal>(),
It.IsAny <PartitionKey?>(),
It.IsAny <Stream>(),
It.IsAny <Action <RequestMessage> >(),
It.IsAny <CosmosDiagnosticsContext>(),
It.IsAny <CancellationToken>()))
.Returns(Task.FromResult(responseMessage));
ContainerInternal containerCore = Mock.Of <ContainerInternal>();
Mock.Get(containerCore)
.Setup(c => c.ClientContext)
.Returns(cosmosClientContext.Object);
FeedRangeInternal range = Mock.Of <FeedRangeInternal>();
Mock.Get(range)
.Setup(f => f.Accept(It.IsAny <FeedRangeRequestMessagePopulatorVisitor>()));
FeedRangeContinuation feedToken = Mock.Of <FeedRangeContinuation>();
Mock.Get(feedToken)
.Setup(f => f.FeedRange)
.Returns(range);
Mock.Get(feedToken)
.Setup(f => f.HandleSplitAsync(It.Is <ContainerInternal>(c => c == containerCore), It.IsAny <ResponseMessage>(), It.IsAny <CancellationToken>()))
.Returns(Task.FromResult(Documents.ShouldRetryResult.NoRetry()));
Mock.Get(feedToken)
.Setup(f => f.GetContinuation())
.Returns(continuation);
Mock.Get(feedToken)
.Setup(f => f.IsDone)
.Returns(true);
FeedRangeIteratorCore feedTokenIterator = new FeedRangeIteratorCore(containerCore, feedToken, new QueryRequestOptions(), Documents.ResourceType.Document, queryDefinition: null);
bool creatorCalled = false;
Func <ResponseMessage, FeedResponse <dynamic> > creator = (ResponseMessage r) =>
{
creatorCalled = true;
return(Mock.Of <FeedResponse <dynamic> >());
};
FeedIteratorCore <dynamic> feedTokenIteratorOfT = new FeedIteratorCore <dynamic>(feedTokenIterator, creator);
FeedResponse <dynamic> response = await feedTokenIteratorOfT.ReadNextAsync();
Assert.IsTrue(creatorCalled, "Response creator not called");
Mock.Get(feedToken)
.Verify(f => f.ReplaceContinuation(It.Is <string>(ct => ct == continuation)), Times.Once);
Mock.Get(feedToken)
.Verify(f => f.HandleSplitAsync(It.Is <ContainerInternal>(c => c == containerCore), It.IsAny <ResponseMessage>(), It.IsAny <CancellationToken>()), Times.Once);
}
public void ReadFromTrace <T>(FeedResponse <T> Response, QueryStatisticsDatumVisitor queryStatisticsDatumVisitor)
{
ITrace trace = ((CosmosTraceDiagnostics)Response.Diagnostics).Value;
//POCO Materialization occurs once per iteration including all the roundtrips
List <ITrace> retrieveQueryMetricTraces = this.FindQueryMetrics(trace: trace, nodeNameOrKeyName: ClientParseTimeNode, isKeyName: false);
foreach (ITrace queryMetricTrace in retrieveQueryMetricTraces)
{
queryStatisticsDatumVisitor.AddPocoTime(queryMetricTrace.Duration.TotalMilliseconds);
}
//Get Cosmos Element Response occurs once per roundtrip for calls with status code 200
List <ITrace> retrieveCosmosElementTraces = this.FindQueryMetrics(trace: trace, nodeNameOrKeyName: ClientDeserializationTimeNode, isKeyName: false);
//Query metrics occurs once per roundtrip for calls with status code 200
List <ITrace> backendMetrics = this.FindQueryMetrics(trace: trace, nodeNameOrKeyName: BackendKeyValue, isKeyName: true);
//Client metrics occurs once per roundtrip for all status codes
List <ITrace> transitMetrics = this.FindQueryMetrics(trace: trace, nodeNameOrKeyName: TransportKeyValue, isKeyName: true, currentNodeName: TransportNodeName);
List <Tuple <ITrace, ITrace, ITrace> > backendAndClientMetrics = new();
int i = 0;
int j = 0;
int k = 0;
foreach (ITrace node in transitMetrics)
{
Debug.Assert(node.Data.Count == 1, "Exactly one transit metric expected");
KeyValuePair <string, object> kvp = node.Data.Single();
Assert.IsInstanceOfType(kvp.Value, typeof(ClientSideRequestStatisticsTraceDatum));
ClientSideRequestStatisticsTraceDatum clientSideRequestStatisticsTraceDatum = (ClientSideRequestStatisticsTraceDatum)kvp.Value;
foreach (ClientSideRequestStatisticsTraceDatum.StoreResponseStatistics storeResponse in clientSideRequestStatisticsTraceDatum.StoreResponseStatisticsList)
{
if (storeResponse.StoreResult.StatusCode == StatusCodes.Ok)
{
backendAndClientMetrics.Add(Tuple.Create(retrieveCosmosElementTraces[k], backendMetrics[j], transitMetrics[i]));
j++;
k++;
}
else
{
//We add null values to the tuple since status codes other than Ok will not have data for 'Query Metrics' and 'Get Cosmos Element Response'
backendAndClientMetrics.Add(Tuple.Create <ITrace, ITrace, ITrace>(null, null, transitMetrics[i]));
}
}
i++;
}
Debug.Assert(i == transitMetrics.Count, "All 'transit metrics' must be grouped.");
Debug.Assert(j == backendMetrics.Count, "All 'backend metrics' must be grouped.");
Debug.Assert(k == retrieveCosmosElementTraces.Count, "All 'Get Cosmos Element Response' traces must be grouped.");
int l = 1;
foreach (Tuple <ITrace, ITrace, ITrace> metrics in backendAndClientMetrics)
{
if (metrics.Item2 != null)
{
Debug.Assert(metrics.Item1 == null, "'Get Cosmos Element Response' is null");
queryStatisticsDatumVisitor.AddGetCosmosElementResponseTime(metrics.Item1.Duration.TotalMilliseconds);
foreach (KeyValuePair <string, object> kvp in metrics.Item2.Data)
{
switch (kvp.Value)
{
case TraceDatum traceDatum:
traceDatum.Accept(queryStatisticsDatumVisitor);
break;
default:
Debug.Fail("Unexpected type", $"Type not supported {metrics.Item2.GetType()}");
break;
}
}
//add metrics to the list except for last roundtrip which is taken care of in ContentSerializationPerformanceTest class
if (l != backendMetrics.Count)
{
queryStatisticsDatumVisitor.PopulateMetrics();
}
l++;
}
foreach (KeyValuePair <string, object> kvp in metrics.Item3.Data)
{
switch (kvp.Value)
{
case TraceDatum traceDatum:
traceDatum.Accept(queryStatisticsDatumVisitor);
break;
default:
Debug.Fail("Unexpected type", $"Type not supported {metrics.Item3.GetType()}");
break;
}
}
}
}
public async Task QueryRequestRateTest(bool directMode)
{
string firstItemIdAndPk = "BasicQueryItem" + Guid.NewGuid();
// Prevent the test from changing the static client
{
CosmosClient client = directMode ? DirectCosmosClient : GatewayCosmosClient;
Container container = client.GetContainer(DatabaseId, ContainerId);
List <string> createdIds = new List <string>()
{
firstItemIdAndPk,
"BasicQueryItem2" + Guid.NewGuid(),
"BasicQueryItem3" + Guid.NewGuid()
};
foreach (string id in createdIds)
{
dynamic item = new
{
id = id,
pk = id,
};
await container.CreateItemAsync <dynamic>(item : item);
}
}
CosmosClient clientWithThrottle;
if (directMode)
{
clientWithThrottle = TestCommon.CreateCosmosClient();
}
else
{
clientWithThrottle = TestCommon.CreateCosmosClient((builder) => builder.WithConnectionModeGateway());
}
Container containerWithThrottle = clientWithThrottle.GetContainer(DatabaseId, ContainerId);
// Do a read to warm up all the caches to prevent them from getting the throttle errors
using (await containerWithThrottle.ReadItemStreamAsync(firstItemIdAndPk, new PartitionKey(firstItemIdAndPk))) { }
Documents.IStoreModel storeModel = clientWithThrottle.ClientContext.DocumentClient.StoreModel;
Mock <Documents.IStoreModel> mockStore = new Mock <Documents.IStoreModel>();
clientWithThrottle.ClientContext.DocumentClient.StoreModel = mockStore.Object;
// Cause 429 after the first call
int callCount = 0;
string activityId = null;
string errorMessage = "QueryRequestRateTest Resource Not Found";
mockStore.Setup(x => x.ProcessMessageAsync(It.IsAny <Documents.DocumentServiceRequest>(), It.IsAny <CancellationToken>()))
.Returns <Documents.DocumentServiceRequest, CancellationToken>((dsr, token) =>
{
callCount++;
if (callCount > 1)
{
INameValueCollection headers = new DictionaryNameValueCollection();
headers.Add(Documents.HttpConstants.HttpHeaders.RetryAfterInMilliseconds, "42");
activityId = Guid.NewGuid().ToString();
headers.Add(Documents.HttpConstants.HttpHeaders.ActivityId, activityId);
Documents.DocumentServiceResponse response = new Documents.DocumentServiceResponse(
body: TestCommon.GenerateStreamFromString(@"{""Errors"":[""" + errorMessage + @"""]}"),
headers: headers,
statusCode: (HttpStatusCode)429,
clientSideRequestStatistics: dsr.RequestContext.ClientRequestStatistics);
return(Task.FromResult(response));
}
return(storeModel.ProcessMessageAsync(dsr, token));
});
List <dynamic> results = new List <dynamic>();
try
{
FeedIterator <dynamic> feedIterator = containerWithThrottle.GetItemQueryIterator <dynamic>(
"select * from T where STARTSWITH(T.id, \"BasicQueryItem\")",
requestOptions: new QueryRequestOptions()
{
MaxItemCount = 1,
MaxConcurrency = 1
});
while (feedIterator.HasMoreResults)
{
FeedResponse <dynamic> response = await feedIterator.ReadNextAsync();
Assert.IsTrue(response.Count <= 1);
Assert.IsTrue(response.Resource.Count() <= 1);
results.AddRange(response);
}
Assert.Fail("Should throw 429 exception after the first page.");
}
catch (CosmosException ce)
{
Assert.IsTrue(ce.RetryAfter.HasValue);
Assert.AreEqual(42, ce.RetryAfter.Value.TotalMilliseconds);
Assert.AreEqual(activityId, ce.ActivityId);
Assert.IsNotNull(ce.DiagnosticsContext);
Assert.IsTrue(ce.Message.Contains(errorMessage));
}
callCount = 0;
FeedIterator streamIterator = containerWithThrottle.GetItemQueryStreamIterator(
"select * from T where STARTSWITH(T.id, \"BasicQueryItem\")",
requestOptions: new QueryRequestOptions()
{
MaxItemCount = 1,
MaxConcurrency = 1
});
// First request should be a success
using (ResponseMessage response = await streamIterator.ReadNextAsync())
{
response.EnsureSuccessStatusCode();
Assert.IsNotNull(response.Content);
}
// Second page should be a failure
using (ResponseMessage response = await streamIterator.ReadNextAsync())
{
Assert.AreEqual(429, (int)response.StatusCode);
Assert.AreEqual("42", response.Headers.RetryAfterLiteral);
Assert.AreEqual(activityId, response.Headers.ActivityId);
Assert.IsNotNull(response.DiagnosticsContext);
Assert.IsTrue(response.ErrorMessage.Contains(errorMessage));
}
}
private FeedResponse <T> ProcessResponse <T>(FeedResponse <T> response)
{
AddSessionTokenToResponseHeaders(response.SessionToken);
return(response);
}
internal virtual async Task <int> QueryAndVerifyDocuments(DocumentClient client, string collectionLink, IEnumerable <Query> queries, int pageSize = 1000, int retries = 0, bool allowScan = false)
{
// First we make sure that all the queries are inserted
{
List <dynamic> queriedDocuments = new List <dynamic>();
IDocumentQuery <Document> selectAllQuery = client.CreateDocumentQuery(collectionLink, feedOptions: new FeedOptions {
MaxItemCount = pageSize, EnableScanInQuery = allowScan, EnableCrossPartitionQuery = true
}).AsDocumentQuery();
while (selectAllQuery.HasMoreResults)
{
FeedResponse <dynamic> queryResultsPage = await selectAllQuery.ExecuteNextAsync();
System.Diagnostics.Trace.TraceInformation("ReadFeed continuation token: {0}, SessionToken: {1}", queryResultsPage.ResponseContinuation, queryResultsPage.SessionToken);
queriedDocuments.AddRange(queryResultsPage);
}
List <dynamic> expected = new List <dynamic>(documents.Count());
for (int i = 0; i < documents.Count(); ++i)
{
expected.Add(JsonConvert.DeserializeObject(String.Format(CultureInfo.InvariantCulture, DocumentFormat, i + 1, String.Empty)));
}
queriedDocuments.Sort((doc1, doc2) => int.Parse(doc1.id).CompareTo(int.Parse(doc2.id)));
var expectedIds = expected.Select(doc => doc.id.ToString());
var actualIds = queriedDocuments.Select(doc => doc.id.ToString());
if (!expectedIds.SequenceEqual(actualIds))
{
System.Diagnostics.Trace.TraceInformation("Failed to insert all the documents, queried documents are:" + Environment.NewLine + String.Join(Environment.NewLine, queriedDocuments));
return(-1);
}
System.Diagnostics.Trace.TraceInformation("All the documents are inserted");
}
// Query and verify
TimeSpan totalQueryLatencyAllPages = TimeSpan.FromSeconds(0);
uint numberOfQueries = 0;
var failedQueries = new List <Query>();
List <Query> query_list = queries as List <Query> ?? queries.ToList();
foreach (var query in query_list)
{
var queriedDocuments = new List <string>();
List <string> activityIDsAllQueryPages = new List <string>();
if (numberOfQueries > 0 && numberOfQueries % 100 == 0)
{
System.Diagnostics.Trace.TraceInformation(DateTime.Now.ToString("HH:mm:ss.ffff") + @": Executing query {0} of {1}",
(numberOfQueries + 1), query_list.Count());
System.Diagnostics.Trace.TraceInformation(@" Query latency per query (avg ms) {0} after {1} queries",
totalQueryLatencyAllPages.TotalMilliseconds / numberOfQueries, numberOfQueries);
}
IDocumentQuery <dynamic> docQuery = client.CreateDocumentQuery(collectionLink, query.ToString(), feedOptions: new FeedOptions {
MaxItemCount = pageSize, EnableScanInQuery = allowScan, EnableCrossPartitionQuery = true
}).AsDocumentQuery();
while (docQuery.HasMoreResults)
{
DateTime startTime = DateTime.Now;
FeedResponse <dynamic> queryResultsPage = await QueryWithRetry(docQuery, query.ToString());
activityIDsAllQueryPages.Add(queryResultsPage.ActivityId);
totalQueryLatencyAllPages += (DateTime.Now - startTime);
foreach (JObject result in queryResultsPage)
{
queriedDocuments.Add(result.ToString(Formatting.None));
}
}
numberOfQueries++;
bool valid;
var expected = Validate(queriedDocuments, query, out valid);
if (!valid)
{
System.Diagnostics.Trace.TraceInformation(
DateTime.Now.ToString("HH:mm:ss.ffff")
+ @": Query {0} did not retrieve expected documents, query all pages activitiIDs: ({1})"
+ Environment.NewLine
+ "Expected:"
+ Environment.NewLine
+ "{2}"
+ Environment.NewLine
+ "Actual:"
+ Environment.NewLine
+ "{3}"
+ Environment.NewLine,
query.ToString(),
String.Join(",", activityIDsAllQueryPages),
String.Join(",", expected),
String.Join(",", queriedDocuments));
failedQueries.Add(query);
}
}
if (failedQueries.Count() == 0)
{
System.Diagnostics.Trace.TraceInformation(@"*** TEST PASSED ***");
return(0);
}
else
{
System.Diagnostics.Trace.TraceInformation(@"*** TEST FAILED with seed {0}***", seed);
int result = -1;
//In case of a failure, retry only failed queries after sleeping for couple of minutes.
if (retries > 0)
{
System.Diagnostics.Trace.TraceInformation(@"*** Retrying Failed queries, {0} retries left ***", --retries);
Task.Delay(120 * 1000).Wait();
result = await QueryAndVerifyDocuments(client, collectionLink, failedQueries, pageSize, retries, allowScan);
}
return(result);
}
}
public async Task TestDistinct_ExecuteNextAsync()
{
async Task ImplementationAsync(Container container, IReadOnlyList <CosmosObject> documents)
{
#region Queries
// To verify distint queries you can run it once without the distinct clause and run it through a hash set
// then compare to the query with the distinct clause.
List <(string, bool)> queries = new List <(string, bool)>()
{
// basic distinct queries
("SELECT {0} VALUE null", true),
// number value distinct queries
("SELECT {0} VALUE c.income from c", true),
// string value distinct queries
("SELECT {0} VALUE c.name from c", true),
// array value distinct queries
("SELECT {0} VALUE c.children from c", true),
// object value distinct queries
("SELECT {0} VALUE c.pet from c", true),
// scalar expressions distinct query
("SELECT {0} VALUE c.age % 2 FROM c", true),
// distinct queries with order by
("SELECT {0} VALUE c.age FROM c ORDER BY c.age", false),
// distinct queries with top and no matching order by
("SELECT {0} TOP 2147483647 VALUE c.age FROM c", false),
// distinct queries with top and matching order by
("SELECT {0} TOP 2147483647 VALUE c.age FROM c ORDER BY c.age", false),
// distinct queries with aggregates
("SELECT {0} VALUE MAX(c.age) FROM c", false),
// distinct queries with joins
("SELECT {0} VALUE c.age FROM p JOIN c IN p.children", true),
// distinct queries in subqueries
("SELECT {0} r.age, s FROM r JOIN (SELECT DISTINCT VALUE c FROM (SELECT 1 a) c) s WHERE r.age > 25", false),
// distinct queries in scalar subqeries
("SELECT {0} p.name, (SELECT DISTINCT VALUE p.age) AS Age FROM p", true),
// select *
("SELECT {0} * FROM c", true)
};
#endregion
#region ExecuteNextAsync API
// run the query with distinct and without + MockDistinctMap
// Should receive same results
// PageSize = 1 guarantees that the backend will return some duplicates.
foreach ((string query, bool allowDCount) in queries)
{
string queryWithoutDistinct = string.Format(query, "");
QueryRequestOptions requestOptions = new QueryRequestOptions()
{
MaxItemCount = 100, MaxConcurrency = 100
};
FeedIterator <CosmosElement> documentQueryWithoutDistinct = container.GetItemQueryIterator <CosmosElement>(
queryWithoutDistinct,
requestOptions: requestOptions);
MockDistinctMap documentsSeen = new MockDistinctMap();
List <CosmosElement> documentsFromWithoutDistinct = new List <CosmosElement>();
while (documentQueryWithoutDistinct.HasMoreResults)
{
FeedResponse <CosmosElement> cosmosQueryResponse = await documentQueryWithoutDistinct.ReadNextAsync();
foreach (CosmosElement document in cosmosQueryResponse)
{
if (documentsSeen.Add(document, out UInt128 hash))
{
documentsFromWithoutDistinct.Add(document);
}
else
{
// No Op for debugging purposes.
}
}
}
foreach (int pageSize in new int[] { 1, 10, 100 })
{
string queryWithDistinct = string.Format(query, "DISTINCT");
List <CosmosElement> documentsFromWithDistinct = new List <CosmosElement>();
FeedIterator <CosmosElement> documentQueryWithDistinct = container.GetItemQueryIterator <CosmosElement>(
queryWithDistinct,
requestOptions: requestOptions);
while (documentQueryWithDistinct.HasMoreResults)
{
FeedResponse <CosmosElement> cosmosQueryResponse = await documentQueryWithDistinct.ReadNextAsync();
documentsFromWithDistinct.AddRange(cosmosQueryResponse);
}
Assert.AreEqual(documentsFromWithDistinct.Count, documentsFromWithoutDistinct.Count);
for (int i = 0; i < documentsFromWithDistinct.Count; i++)
{
CosmosElement documentFromWithDistinct = documentsFromWithDistinct.ElementAt(i);
CosmosElement documentFromWithoutDistinct = documentsFromWithoutDistinct.ElementAt(i);
Assert.AreEqual(
expected: documentFromWithoutDistinct,
actual: documentFromWithDistinct,
message: $"{documentFromWithDistinct} did not match {documentFromWithoutDistinct} at index {i} for {queryWithDistinct}, with page size: {pageSize} on a container");
}
if (allowDCount)
{
string queryWithDCount = $"SELECT VALUE COUNT(1) FROM({queryWithDistinct})";
List <CosmosElement> documentsWithDCount = new List <CosmosElement>();
FeedIterator <CosmosElement> documentQueryWithDCount = container.GetItemQueryIterator <CosmosElement>(
queryWithDCount,
requestOptions: requestOptions);
while (documentQueryWithDCount.HasMoreResults)
{
FeedResponse <CosmosElement> cosmosQueryResponse = await documentQueryWithDCount.ReadNextAsync();
documentsWithDCount.AddRange(cosmosQueryResponse);
}
Assert.AreEqual(1, documentsWithDCount.Count);
long dcount = Number64.ToLong((documentsWithDCount.First() as CosmosNumber).Value);
Assert.AreEqual(documentsFromWithoutDistinct.Count, dcount);
}
}
}
#endregion
}
await this.TestQueryDistinctBaseAsync(ImplementationAsync);
}
public void ValidateResponseFactoryJsonSerializer()
{
ResponseMessage databaseResponse = this.CreateResponse();
ResponseMessage containerResponse = this.CreateResponse();
ResponseMessage storedProcedureExecuteResponse = this.CreateResponse();
ResponseMessage storedProcedureResponse = this.CreateResponse();
ResponseMessage triggerResponse = this.CreateResponse();
ResponseMessage udfResponse = this.CreateResponse();
ResponseMessage itemResponse = this.CreateResponse();
Mock <CosmosSerializer> mockUserJsonSerializer = new Mock <CosmosSerializer>();
CosmosSerializerCore serializerCore = new CosmosSerializerCore(mockUserJsonSerializer.Object);
CosmosResponseFactoryInternal cosmosResponseFactory = new CosmosResponseFactoryCore(
serializerCore);
// Test the user specified response
mockUserJsonSerializer.Setup(x => x.FromStream <ToDoActivity>(itemResponse.Content)).Callback <Stream>(input => input.Dispose()).Returns(new ToDoActivity());
mockUserJsonSerializer.Setup(x => x.FromStream <ToDoActivity>(storedProcedureExecuteResponse.Content)).Callback <Stream>(input => input.Dispose()).Returns(new ToDoActivity());
// Verify all the user types use the user specified version
cosmosResponseFactory.CreateItemResponse <ToDoActivity>(itemResponse);
cosmosResponseFactory.CreateStoredProcedureExecuteResponse <ToDoActivity>(storedProcedureExecuteResponse);
// Throw if the setups were not called
mockUserJsonSerializer.VerifyAll();
// Test read feed scenario
ResponseMessage readFeedResponse = this.CreateReadFeedResponse();
mockUserJsonSerializer.Setup(x => x.FromStream <ToDoActivity[]>(It.IsAny <Stream>()))
.Callback <Stream>(input => input.Dispose())
.Returns(new ToDoActivity[] { new ToDoActivity() });
FeedResponse <ToDoActivity> feedResponse = cosmosResponseFactory.CreateItemFeedResponse <ToDoActivity>(readFeedResponse);
foreach (ToDoActivity toDoActivity in feedResponse)
{
Assert.IsNotNull(toDoActivity);
}
mockUserJsonSerializer.VerifyAll();
ResponseMessage changeFeedResponseMessage = this.CreateChangeFeedNotModifiedResponse();
try
{
cosmosResponseFactory.CreateItemFeedResponse <ToDoActivity>(changeFeedResponseMessage);
Assert.Fail();
}
catch (CosmosException cosmosException)
{
Assert.AreEqual(HttpStatusCode.NotModified, cosmosException.StatusCode);
}
ResponseMessage queryResponse = this.CreateReadFeedResponse();
mockUserJsonSerializer.Setup(x => x.FromStream <ToDoActivity[]>(It.IsAny <Stream>())).Callback <Stream>(input => input.Dispose()).Returns(new ToDoActivity[] { new ToDoActivity() });
FeedResponse <ToDoActivity> queryFeedResponse = cosmosResponseFactory.CreateItemFeedResponse <ToDoActivity>(queryResponse);
foreach (ToDoActivity toDoActivity in queryFeedResponse)
{
Assert.IsNotNull(toDoActivity);
}
mockUserJsonSerializer.VerifyAll();
// Test the system specified response
ContainerProperties containerSettings = new ContainerProperties("mockId", "/pk");
DatabaseProperties databaseSettings = new DatabaseProperties()
{
Id = "mock"
};
StoredProcedureProperties cosmosStoredProcedureSettings = new StoredProcedureProperties()
{
Id = "mock"
};
TriggerProperties cosmosTriggerSettings = new TriggerProperties()
{
Id = "mock"
};
UserDefinedFunctionProperties cosmosUserDefinedFunctionSettings = new UserDefinedFunctionProperties()
{
Id = "mock"
};
Mock <Container> mockContainer = new Mock <Container>();
Mock <Database> mockDatabase = new Mock <Database>();
// Verify all the system types that should always use default
cosmosResponseFactory.CreateContainerResponse(mockContainer.Object, containerResponse);
cosmosResponseFactory.CreateDatabaseResponse(mockDatabase.Object, databaseResponse);
cosmosResponseFactory.CreateStoredProcedureResponse(storedProcedureResponse);
cosmosResponseFactory.CreateTriggerResponse(triggerResponse);
cosmosResponseFactory.CreateUserDefinedFunctionResponse(udfResponse);
}
//Helper method to run query
static async Task RunQuery(string sqlQueryText, int maxItemCountPerPage = 100, int maxConcurrency = -1, bool useQueryOptions = false)
{
Console.BackgroundColor = ConsoleColor.Blue;
Console.WriteLine($"Running query: \"{sqlQueryText}\" against container {ContainerName}\n");
Console.WriteLine("");
if (useQueryOptions)
{
Console.WriteLine($"Using MaxConcurrency: {maxConcurrency}");
Console.WriteLine($"Using MaxItemCountPerPage: {maxItemCountPerPage}");
}
Console.ResetColor();
double totalRequestCharge = 0;
QueryDefinition queryDefinition = new QueryDefinition(sqlQueryText);
// Run query against Cosmos DB
var container = cosmosClient.GetDatabase(DatabaseName).GetContainer(ContainerName);
QueryRequestOptions requestOptions;
if (useQueryOptions)
{
requestOptions = new QueryRequestOptions()
{
MaxItemCount = maxItemCountPerPage,
MaxConcurrency = maxConcurrency,
};
}
else
{
requestOptions = new QueryRequestOptions(); //use all default query options
}
// Time the query
Stopwatch stopWatch = new Stopwatch();
stopWatch.Start();
FeedIterator <dynamic> queryResultSetIterator = container.GetItemQueryIterator <dynamic>(queryDefinition, requestOptions: requestOptions);
List <dynamic> reviews = new List <dynamic>();
while (queryResultSetIterator.HasMoreResults)
{
FeedResponse <dynamic> currentResultSet = await queryResultSetIterator.ReadNextAsync();
totalRequestCharge += currentResultSet.RequestCharge;
//Console.WriteLine("another page");
foreach (var item in currentResultSet)
{
reviews.Add(item);
Console.WriteLine(item);
}
if (useQueryOptions)
{
Console.WriteLine($"Result count: {reviews.Count}");
}
}
stopWatch.Stop();
TimeSpan ts = stopWatch.Elapsed;
//Print results
string elapsedTime = String.Format("{0:00}:{1:00}:{2:00}.{3:00}",
ts.Hours, ts.Minutes, ts.Seconds,
ts.Milliseconds / 10);
Console.ForegroundColor = ConsoleColor.Green;
Console.WriteLine($"\tQuery returned {reviews.Count} results");
Console.WriteLine($"\tTotal time: {elapsedTime}");
Console.WriteLine($"\tTotal Request Units consumed: {totalRequestCharge}\n");
Console.WriteLine("\n\n\n");
Console.ResetColor();
}
private async Task <long> ExecuteQueryAndReturnOutputDocumentCount(
string queryText,
int expectedItemCount,
bool disableDiagnostics)
{
QueryDefinition sql = null;
if (queryText != null)
{
sql = new QueryDefinition(queryText);
}
QueryRequestOptions requestOptions = new QueryRequestOptions()
{
MaxItemCount = 1,
MaxConcurrency = 1,
};
if (disableDiagnostics)
{
requestOptions.DiagnosticContextFactory = () => EmptyCosmosDiagnosticsContext.Singleton;
}
;
// Verify the typed query iterator
FeedIterator <ToDoActivity> feedIterator = this.Container.GetItemQueryIterator <ToDoActivity>(
sql,
requestOptions: requestOptions);
List <ToDoActivity> results = new List <ToDoActivity>();
long totalOutDocumentCount = 0;
bool isFirst = true;
while (feedIterator.HasMoreResults)
{
FeedResponse <ToDoActivity> response = await feedIterator.ReadNextAsync();
results.AddRange(response);
if (queryText == null)
{
CosmosDiagnosticsTests.VerifyPointDiagnostics(
response.Diagnostics,
disableDiagnostics);
}
else
{
VerifyQueryDiagnostics(
response.Diagnostics,
isFirst,
disableDiagnostics);
}
isFirst = false;
}
Assert.AreEqual(expectedItemCount, results.Count);
// Verify the stream query iterator
FeedIterator streamIterator = this.Container.GetItemQueryStreamIterator(
sql,
requestOptions: requestOptions);
List <ToDoActivity> streamResults = new List <ToDoActivity>();
long streamTotalOutDocumentCount = 0;
isFirst = true;
while (streamIterator.HasMoreResults)
{
ResponseMessage response = await streamIterator.ReadNextAsync();
Collection <ToDoActivity> result = TestCommon.SerializerCore.FromStream <CosmosFeedResponseUtil <ToDoActivity> >(response.Content).Data;
streamResults.AddRange(result);
if (queryText == null)
{
CosmosDiagnosticsTests.VerifyPointDiagnostics(
response.Diagnostics,
disableDiagnostics);
}
else
{
VerifyQueryDiagnostics(
response.Diagnostics,
isFirst,
disableDiagnostics);
}
isFirst = false;
}
Assert.AreEqual(expectedItemCount, streamResults.Count);
Assert.AreEqual(totalOutDocumentCount, streamTotalOutDocumentCount);
return(results.Count);
}
private async Task DataUpdateStepAsync(WaterfallStepContext stepContext, CancellationToken cancellationToken)
{
var orderData = await _orderDataAccessor.GetAsync(stepContext.Context, () => new OrderData(), cancellationToken);
CosmosClient client = new CosmosClient(Startup.CosmosDbEndpoint, Startup.AuthKey);
database = await client.CreateDatabaseIfNotExistsAsync(databaseId);
ContainerProperties containerProperties = new ContainerProperties(containerId, partitionKeyPath: Startup.PartitionKey);
// Create with a throughput of 1000 RU/s
container = await database.CreateContainerIfNotExistsAsync(
containerProperties,
throughput : 1000);
//orderNum 가져오기
try
{
FeedIterator <DBdata> feedIterator = container.GetItemQueryIterator <DBdata>("SELECT top 1 * FROM c order by c._ts desc");
{
while (feedIterator.HasMoreResults)
{
FeedResponse <DBdata> result = await feedIterator.ReadNextAsync();
foreach (var item in result)
{
orderData.OrderNum = Int32.Parse(item.id);
}
}
}
}
catch (System.Exception e)
{
orderData.OrderNum = 0;
}
//ip
IPHostEntry host = Dns.GetHostEntry(Dns.GetHostName());
string ipAddr = string.Empty;
for (int i = 0; i < host.AddressList.Length; i++)
{
if (host.AddressList[i].AddressFamily == AddressFamily.InterNetwork)
{
ipAddr = host.AddressList[i].ToString();
}
}
//Order 데이터 삽입
foreach (Sandwich tempSand in orderData.Sandwiches)
{
string sJson = JsonConvert.SerializeObject(tempSand);
var dbData = new DBdata {
id = $"{++orderData.OrderNum}", Contents = tempSand, ETag = "x", AccountNumber = ipAddr
};
await container.CreateItemAsync <DBdata>(dbData, new PartitionKey(dbData.AccountNumber));
}
//container 변경
containerProperties = new ContainerProperties(countContainerId, partitionKeyPath: "/AccountNumber");
// Create with a throughput of 1000 RU/s
container = await database.CreateContainerIfNotExistsAsync(
containerProperties,
throughput : 1000);
//Count 데이터 삽입
foreach (Sandwich tempSand in orderData.Sandwiches)
{
var CountId = 0;
string sauce = "";
tempSand.Sauce.Sort();
foreach (string temp in tempSand.Sauce)
{
sauce += temp + " ";
}
//CountId 찾기
try
{
FeedIterator <DBcount> feedIterator = container.GetItemQueryIterator <DBcount>("SELECT top 1 * FROM c order by c._ts desc");
{
while (feedIterator.HasMoreResults)
{
FeedResponse <DBcount> result = await feedIterator.ReadNextAsync();
foreach (var item in result)
{
CountId = Int32.Parse(item.id) + 1;
}
}
}
}
catch (System.Exception e)
{
CountId = 0;
}
try {
FeedIterator <DBcount> feedIterator = container.GetItemQueryIterator <DBcount>("SELECT * FROM c WHERE c.Sauce='" + sauce + "' and c.Bread ='" + tempSand.Bread + "' and c.Menu ='" + tempSand.Menu + "'");
{
if (feedIterator.HasMoreResults)
{
FeedResponse <DBcount> result = await feedIterator.ReadNextAsync();
DBcount res = result.First();
var count = res.Count + 1;
DBcount countData = new DBcount(); countData.id = res.id; countData.Count = count; countData.Sauce = sauce; countData.Menu = tempSand.Menu; countData.ETag = "x"; countData.AccountNumber = "0"; countData.Bread = tempSand.Bread;
ItemResponse <DBcount> item = await container.DeleteItemAsync <DBcount>(partitionKey : new PartitionKey("0"), id : res.id);
await container.CreateItemAsync(countData, new PartitionKey("0"));
}
}
}
catch (System.Exception e)
{
var countData = new DBcount {
id = $"{CountId}", Count = 1, Bread = $"{tempSand.Bread}", Sauce = sauce, Menu = $"{tempSand.Menu}", ETag = "x", AccountNumber = "0"
};
await container.CreateItemAsync <DBcount>(countData, new PartitionKey("0"));
}
}
}
private async Task <FeedResponse <CosmosElement> > ExecuteOnceAsync(IDocumentClientRetryPolicy retryPolicyInstance, CancellationToken cancellationToken)
{
// Don't reuse request, as the rest of client SDK doesn't reuse requests between retries.
// The code leaves some temporary garbage in request (in RequestContext etc.),
// which shold be erased during retries.
using (DocumentServiceRequest request = await this.CreateRequestAsync())
{
if (!string.IsNullOrEmpty(request.Headers[HttpConstants.HttpHeaders.PartitionKey]) ||
!request.ResourceType.IsPartitioned())
{
return(await this.ExecuteRequestAsync(request, cancellationToken));
}
CollectionCache collectionCache = await this.Client.GetCollectionCacheAsync();
CosmosContainerSettings collection =
await collectionCache.ResolveCollectionAsync(request, CancellationToken.None);
if (!string.IsNullOrEmpty(base.PartitionKeyRangeId))
{
request.RouteTo(new PartitionKeyRangeIdentity(collection.ResourceId, base.PartitionKeyRangeId));
return(await this.ExecuteRequestAsync(request, cancellationToken));
}
// For non-Windows platforms(like Linux and OSX) in .NET Core SDK, we cannot use ServiceInterop for parsing the query,
// so forcing the request through Gateway. We are also now by-passing this for 32-bit host process in NETFX on Windows
// as the ServiceInterop dll is only available in 64-bit.
if (CustomTypeExtensions.ByPassQueryParsing())
{
request.UseGatewayMode = true;
return(await this.ExecuteRequestAsync(request, cancellationToken));
}
QueryPartitionProvider queryPartitionProvider = await this.Client.GetQueryPartitionProviderAsync(cancellationToken);
IRoutingMapProvider routingMapProvider = await this.Client.GetRoutingMapProviderAsync();
List <CompositeContinuationToken> suppliedTokens;
Range <string> rangeFromContinuationToken =
this.partitionRoutingHelper.ExtractPartitionKeyRangeFromContinuationToken(request.Headers, out suppliedTokens);
Tuple <PartitionRoutingHelper.ResolvedRangeInfo, IReadOnlyList <Range <string> > > queryRoutingInfo =
await this.TryGetTargetPartitionKeyRangeAsync(
request,
collection,
queryPartitionProvider,
routingMapProvider,
rangeFromContinuationToken,
suppliedTokens);
if (request.IsNameBased && queryRoutingInfo == null)
{
request.ForceNameCacheRefresh = true;
collection = await collectionCache.ResolveCollectionAsync(request, CancellationToken.None);
queryRoutingInfo = await this.TryGetTargetPartitionKeyRangeAsync(
request,
collection,
queryPartitionProvider,
routingMapProvider,
rangeFromContinuationToken,
suppliedTokens);
}
if (queryRoutingInfo == null)
{
throw new NotFoundException($"{DateTime.UtcNow.ToString("o", CultureInfo.InvariantCulture)}: Was not able to get queryRoutingInfo even after resolve collection async with force name cache refresh to the following collectionRid: {collection.ResourceId} with the supplied tokens: {JsonConvert.SerializeObject(suppliedTokens)}");
}
request.RouteTo(new PartitionKeyRangeIdentity(collection.ResourceId, queryRoutingInfo.Item1.ResolvedRange.Id));
FeedResponse <CosmosElement> response = await this.ExecuteRequestLazyAsync(request, cancellationToken);
if (!await this.partitionRoutingHelper.TryAddPartitionKeyRangeToContinuationTokenAsync(
response.Headers,
providedPartitionKeyRanges: queryRoutingInfo.Item2,
routingMapProvider: routingMapProvider,
collectionRid: collection.ResourceId,
resolvedRangeInfo: queryRoutingInfo.Item1))
{
// Collection to which this request was resolved doesn't exist.
// Retry policy will refresh the cache and return NotFound.
throw new NotFoundException($"{DateTime.UtcNow.ToString("o", CultureInfo.InvariantCulture)}: Call to TryAddPartitionKeyRangeToContinuationTokenAsync failed to the following collectionRid: {collection.ResourceId} with the supplied tokens: {JsonConvert.SerializeObject(suppliedTokens)}");
}
return(response);
}
}
/// <summary>
/// Exposing progress with the Estimator with the detailed iterator.
/// </summary>
/// <remarks>
/// The Estimator uses the same processorName and the same lease configuration as the existing processor to measure progress.
/// The iterator exposes detailed, per-lease, information on estimation and ownership.
/// </remarks>
public static async Task RunEstimatorPullChangeFeed(
string databaseId,
CosmosClient client)
{
await Program.InitializeAsync(databaseId, client);
// <StartProcessorEstimatorDetailed>
Container leaseContainer = client.GetContainer(databaseId, Program.leasesContainer);
Container monitoredContainer = client.GetContainer(databaseId, Program.monitoredContainer);
ChangeFeedProcessor changeFeedProcessor = monitoredContainer
.GetChangeFeedProcessorBuilder <ToDoItem>("changeFeedEstimator", Program.HandleChangesAsync)
.WithInstanceName("consoleHost")
.WithLeaseContainer(leaseContainer)
.Build();
// </StartProcessorEstimatorDetailed>
Console.WriteLine($"Starting Change Feed Processor...");
await changeFeedProcessor.StartAsync();
Console.WriteLine("Change Feed Processor started.");
// Wait some seconds for instances to acquire leases
await Task.Delay(5000);
Console.WriteLine("Generating 10 items that will be picked up by the delegate...");
await Program.GenerateItems(10, client.GetContainer(databaseId, Program.monitoredContainer));
// Wait random time for the delegate to output all messages after initialization is done
await Task.Delay(5000);
// <StartEstimatorDetailed>
ChangeFeedEstimator changeFeedEstimator = monitoredContainer
.GetChangeFeedEstimator("changeFeedEstimator", leaseContainer);
// </StartEstimatorDetailed>
// <GetIteratorEstimatorDetailed>
Console.WriteLine("Checking estimation...");
using FeedIterator <ChangeFeedProcessorState> estimatorIterator = changeFeedEstimator.GetCurrentStateIterator();
while (estimatorIterator.HasMoreResults)
{
FeedResponse <ChangeFeedProcessorState> states = await estimatorIterator.ReadNextAsync();
foreach (ChangeFeedProcessorState leaseState in states)
{
string host = leaseState.InstanceName == null ? $"not owned by any host currently" : $"owned by host {leaseState.InstanceName}";
Console.WriteLine($"Lease [{leaseState.LeaseToken}] {host} reports {leaseState.EstimatedLag} as estimated lag.");
}
}
// </GetIteratorEstimatorDetailed>
Console.WriteLine("Stopping processor to show how the lag increases if no processing is happening.");
await changeFeedProcessor.StopAsync();
// Wait for processor to shutdown completely so the next items generate lag
await Task.Delay(5000);
Console.WriteLine("Generating 10 items that will be seen by the Estimator...");
await Program.GenerateItems(10, client.GetContainer(databaseId, Program.monitoredContainer));
Console.WriteLine("Checking estimation...");
using FeedIterator <ChangeFeedProcessorState> estimatorIteratorAfter = changeFeedEstimator.GetCurrentStateIterator();
while (estimatorIteratorAfter.HasMoreResults)
{
FeedResponse <ChangeFeedProcessorState> states = await estimatorIteratorAfter.ReadNextAsync();
foreach (ChangeFeedProcessorState leaseState in states)
{
// Host ownership should be empty as we have already stopped the estimator
string host = leaseState.InstanceName == null ? $"not owned by any host currently" : $"owned by host {leaseState.InstanceName}";
Console.WriteLine($"Lease [{leaseState.LeaseToken}] {host} reports {leaseState.EstimatedLag} as estimated lag.");
}
}
Console.WriteLine("Press any key to continue with the next demo...");
Console.ReadKey();
}
// </RunSimpleScript>
/// <summary>
/// Import many documents using stored procedure.
/// </summary>
// <RunBulkImport>
private static async Task RunBulkImport(Container container)
{
string inputDirectory = @".\Data\";
string inputFileMask = "*.json";
int maxFiles = 2000;
int maxScriptSize = 50000;
// 1. Get the files.
string[] fileNames = Directory.GetFiles(inputDirectory, inputFileMask);
DirectoryInfo di = new DirectoryInfo(inputDirectory);
FileInfo[] fileInfos = di.GetFiles(inputFileMask);
// 2. Prepare for import.
int currentCount = 0;
int fileCount = maxFiles != 0 ? Math.Min(maxFiles, fileNames.Length) : fileNames.Length;
// 3. Create stored procedure for this script.
string scriptId = "BulkImport";
string body = File.ReadAllText(@".\JS\BulkImport.js");
await TryDeleteStoredProcedure(container, scriptId);
Scripts cosmosScripts = container.Scripts;
StoredProcedureResponse sproc = await cosmosScripts.CreateStoredProcedureAsync(new StoredProcedureProperties(scriptId, body));
// 4. Create a batch of docs (MAX is limited by request size (2M) and to script for execution.
// We send batches of documents to create to script.
// Each batch size is determined by MaxScriptSize.
// MaxScriptSize should be so that:
// -- it fits into one request (MAX request size is 16Kb).
// -- it doesn't cause the script to time out.
// -- it is possible to experiment with MaxScriptSize to get best performance given number of throttles, etc.
while (currentCount < fileCount)
{
// 5. Create args for current batch.
// Note that we could send a string with serialized JSON and JSON.parse it on the script side,
// but that would cause script to run longer. Since script has timeout, unload the script as much
// as we can and do the parsing by client and framework. The script will get JavaScript objects.
string argsJson = CreateBulkInsertScriptArguments(fileNames, currentCount, fileCount, maxScriptSize);
dynamic[] args = new dynamic[] { JsonConvert.DeserializeObject <dynamic>(argsJson) };
// 6. execute the batch.
StoredProcedureExecuteResponse <int> scriptResult = await cosmosScripts.ExecuteStoredProcedureAsync <dynamic, int>(new PartitionKey("Andersen"), scriptId, args);
// 7. Prepare for next batch.
int currentlyInserted = scriptResult.Resource;
currentCount += currentlyInserted;
}
// 8. Validate
int numDocs = 0;
FeedIterator <dynamic> setIterator = container.GetItemIterator <dynamic>();
while (setIterator.HasMoreResults)
{
FeedResponse <dynamic> response = await setIterator.ReadNextAsync();
numDocs += response.Count();
}
Console.WriteLine("Found {0} documents in the collection. There were originally {1} files in the Data directory\r\n", numDocs, fileCount);
}
public static void Run([TimerTrigger("0 */5 * * * *")] TimerInfo myTimer, TraceWriter log)
{
_subscriptionId = GetEnvironmentVariable("subscriptionId");
_tenantId = GetEnvironmentVariable("tenantId");
_applicationId = GetEnvironmentVariable("applicationId");
_applicationPwd = GetEnvironmentVariable("applicationPwd");
_resourceGroupName = GetEnvironmentVariable("resourceGroupName");
_accountName = GetEnvironmentVariable("accountName");
_accountUri = GetEnvironmentVariable("accountUri");
_accountKey = GetEnvironmentVariable("authKey");
_dbName = GetEnvironmentVariable("dbName");
_collName = GetEnvironmentVariable("collName");
_maxAllowedThroughput = GetEnvironmentVariable("maxAllowedThroughput");
_minAllowedThroughput = GetEnvironmentVariable("minAllowedThroughput");
_throughputStep = GetEnvironmentVariable("throughputStep");
//Console.WriteLine("accountUri: " + _accountUri);
log.Info($"Timer trigger function executed at: {DateTime.Now}");
docDbClient = new DocumentClient(
new Uri(_accountUri),
_accountKey,
new ConnectionPolicy
{
ConnectionMode = ConnectionMode.Direct,
ConnectionProtocol = Protocol.Tcp
});
DocumentCollection collection = GetCollection(_dbName, _collName);
Uri collectionUri = UriFactory.CreateDocumentCollectionUri(_dbName, collection.Id);
FeedResponse <PartitionKeyRange> pkRanges = docDbClient.ReadPartitionKeyRangeFeedAsync(
collectionUri).Result;
int partitionCount = pkRanges.Count;
log.Info($"The number of partitions in the collection is {partitionCount}");
int provisionedThroughput = GetOfferThroughput(collection.SelfLink);
log.Info($"Provisioned throughtput is {provisionedThroughput}");
int partitionThroughput = provisionedThroughput / partitionCount;
MetricCollection metricList = ReadMetricValues(collection.SelfLink, GetAccessToken());
int throughputIncrease = 0;
bool isCollectionBusy = false;
bool isCollectionOverProvisioned = true;
foreach (var metric in metricList.Value)
{
foreach (var metricValue in metric.MetricValues)
{
log.Info($"Max RUs Per Second is {metricValue.Maximum}. Throughtput per partition is {partitionThroughput}");
if (metricValue.Maximum > partitionThroughput)
{
isCollectionBusy = true;
isCollectionOverProvisioned = false;
log.Info($"Partition throughput increased more than provisioned at {metricValue.Maximum} - {metricValue.Timestamp}");
throughputIncrease = (int)metricValue.Maximum * partitionCount;
break;
}
else if (partitionThroughput > metricValue.Maximum)
{
isCollectionOverProvisioned = true;
}
}
}
if (isCollectionBusy)
{
if (throughputIncrease > Int32.Parse(_maxAllowedThroughput))
{
log.Info($"Max throughput limit reached - {_maxAllowedThroughput}. Cannot scale up further");
ScaleOffer(_dbName, _collName, Int32.Parse(_maxAllowedThroughput));
return;
}
log.Info($"Scaling up collection {_collName} throughput to {throughputIncrease}");
ScaleOffer(_dbName, _collName, throughputIncrease);
}
if (isCollectionOverProvisioned)
{
throughputIncrease = partitionThroughput - Int32.Parse(_throughputStep);
if (Int32.Parse(_minAllowedThroughput) > throughputIncrease)
{
log.Info($"Min throughput limit reached - {_minAllowedThroughput}. Cannot scale down further");
ScaleOffer(_dbName, _collName, Int32.Parse(_minAllowedThroughput));
return;
}
log.Info($"Scaling down collection {_collName} throughput to {throughputIncrease}");
ScaleOffer(_dbName, _collName, throughputIncrease);
}
log.Info($"Completed execution at: {DateTime.Now}");
}
public async Task RunAsync(
CTLConfig config,
CosmosClient cosmosClient,
ILogger logger,
IMetrics metrics,
string loggingContextIdentifier,
CancellationToken cancellationToken)
{
logger.LogInformation("Initializing counters and metrics.");
CounterOptions documentCounter = new CounterOptions {
Name = "#Documents received", Context = loggingContextIdentifier
};
GaugeOptions leaseGauge = new GaugeOptions {
Name = "#Leases created", Context = loggingContextIdentifier
};
string leaseContainerId = Guid.NewGuid().ToString();
Container leaseContainer = await cosmosClient.GetDatabase(config.Database).CreateContainerAsync(leaseContainerId, "/id");
logger.LogInformation("Created lease container {0}", leaseContainerId);
try
{
object lockObject = new object();
long documentTotal = 0;
ChangeFeedProcessor changeFeedProcessor = cosmosClient.GetContainer(config.Database, config.Collection)
.GetChangeFeedProcessorBuilder <SimpleItem>("ctlProcessor",
(IReadOnlyCollection <SimpleItem> docs, CancellationToken token) =>
{
lock (lockObject)
{
documentTotal += docs.Count;
}
metrics.Measure.Counter.Increment(documentCounter, docs.Count);
return(Task.CompletedTask);
})
.WithLeaseContainer(leaseContainer)
.WithInstanceName(Guid.NewGuid().ToString())
.WithErrorNotification((string leaseToken, Exception ex) =>
{
Utils.LogError(logger, loggingContextIdentifier, ex);
return(Task.CompletedTask);
})
.WithStartTime(DateTime.MinValue.ToUniversalTime())
.Build();
await changeFeedProcessor.StartAsync();
logger.LogInformation("Started change feed processor");
await Task.Delay(config.RunningTimeDurationAsTimespan, cancellationToken);
logger.LogInformation("Stopping change feed processor");
await changeFeedProcessor.StopAsync();
// List leases
using FeedIterator <LeaseSchema> leaseIterator = leaseContainer.GetItemQueryIterator <LeaseSchema>();
int leaseTotal = 0;
List <FeedRange> ranges = new List <FeedRange>();
while (leaseIterator.HasMoreResults)
{
FeedResponse <LeaseSchema> response = await leaseIterator.ReadNextAsync();
foreach (LeaseSchema lease in response)
{
if (lease.LeaseToken != null)
{
logger.LogInformation($"Lease for range {lease.LeaseToken} - {lease.FeedRange.EffectiveRange.Min} - {lease.FeedRange.EffectiveRange.Max}");
ranges.Add(lease.FeedRange.EffectiveRange);
leaseTotal++;
}
}
}
logger.LogInformation($"Total count of leases {leaseTotal}.");
metrics.Measure.Gauge.SetValue(leaseGauge, leaseTotal);
string previousMin = "";
foreach (FeedRange sortedRange in ranges.OrderBy(range => range.Min))
{
if (previousMin != sortedRange.Min)
{
Utils.LogError(logger, loggingContextIdentifier,
$"Expected a sorted range with Min <{previousMin}> but encountered range <{sortedRange.Min}>:<{sortedRange.Max}>");
}
previousMin = sortedRange.Max;
}
if (config.PreCreatedDocuments > 0)
{
if (this.initializationResult.InsertedDocuments != documentTotal)
{
Utils.LogError(logger, loggingContextIdentifier, $"Expected to receive {this.initializationResult.InsertedDocuments} documents and got {documentTotal}");
}
}
}
catch (Exception ex)
{
Utils.LogError(logger, loggingContextIdentifier, ex);
}
finally
{
await leaseContainer.DeleteContainerAsync();
if (this.initializationResult.CreatedDatabase)
{
await cosmosClient.GetDatabase(config.Database).DeleteAsync();
}
if (this.initializationResult.CreatedContainer)
{
await cosmosClient.GetContainer(config.Database, config.Collection).DeleteContainerAsync();
}
}
}
public async Task ChangeFeedIteratorCore_PartitionKey_OfT_ReadAll()
{
int totalCount = 0;
int firstRunTotal = 25;
int batchSize = 25;
string pkToRead = "pkToRead";
string otherPK = "otherPK";
for (int i = 0; i < batchSize; i++)
{
await this.Container.CreateItemAsync(this.CreateRandomToDoActivity(pkToRead));
}
for (int i = 0; i < batchSize; i++)
{
await this.Container.CreateItemAsync(this.CreateRandomToDoActivity(otherPK));
}
ContainerInternal itemsCore = this.Container;
FeedIterator <ToDoActivity> feedIterator = itemsCore.GetChangeFeedIterator <ToDoActivity>(
ChangeFeedStartFrom.Beginning(
new FeedRangePartitionKey(
new PartitionKey(pkToRead))),
new ChangeFeedRequestOptions()
{
PageSizeHint = 1,
});
string continuation = null;
while (feedIterator.HasMoreResults)
{
FeedResponse <ToDoActivity> feedResponse = await feedIterator.ReadNextAsync(this.cancellationToken);
totalCount += feedResponse.Count;
foreach (ToDoActivity toDoActivity in feedResponse)
{
Assert.AreEqual(pkToRead, toDoActivity.status);
}
continuation = feedResponse.ContinuationToken;
}
Assert.AreEqual(firstRunTotal, totalCount);
int expectedFinalCount = 50;
// Insert another batch of 25 and use the last FeedToken from the first cycle
for (int i = 0; i < batchSize; i++)
{
await this.Container.CreateItemAsync(this.CreateRandomToDoActivity(pkToRead));
}
FeedIterator <ToDoActivity> setIteratorNew = itemsCore.GetChangeFeedIterator <ToDoActivity>(
ChangeFeedStartFrom.ContinuationToken(continuation));
while (setIteratorNew.HasMoreResults)
{
FeedResponse <ToDoActivity> feedResponse = await setIteratorNew.ReadNextAsync(this.cancellationToken);
totalCount += feedResponse.Count;
foreach (ToDoActivity toDoActivity in feedResponse)
{
Assert.AreEqual(pkToRead, toDoActivity.status);
}
}
Assert.AreEqual(expectedFinalCount, totalCount);
}
public async Task ItemResourcePermissionTest()
{
//create user
string userId = Guid.NewGuid().ToString();
UserResponse userResponse = await this.cosmosDatabase.CreateUserAsync(userId);
Assert.AreEqual(HttpStatusCode.Created, userResponse.StatusCode);
Assert.AreEqual(userId, userResponse.Resource.Id);
User user = userResponse.User;
//create resource
string containerId = Guid.NewGuid().ToString();
ContainerResponse containerResponse = await this.cosmosDatabase.CreateContainerAsync(containerId, "/id");
Assert.AreEqual(HttpStatusCode.Created, containerResponse.StatusCode);
Container container = containerResponse.Container;
string itemId = Guid.NewGuid().ToString();
PartitionKey partitionKey = new PartitionKey(itemId);
ItemResponse <dynamic> itemRespnose = await container.CreateItemAsync <dynamic>(new { id = itemId }, partitionKey);
Assert.AreEqual(HttpStatusCode.Created, itemRespnose.StatusCode);
//create permission
string permissionId = Guid.NewGuid().ToString();
PermissionProperties permissionProperties = new PermissionProperties(permissionId, PermissionMode.Read, container, partitionKey, itemId);
PermissionResponse permissionResponse = await user.CreatePermissionAsync(permissionProperties);
PermissionProperties permission = permissionResponse.Resource;
Assert.AreEqual(HttpStatusCode.Created, userResponse.StatusCode);
Assert.AreEqual(permissionId, permission.Id);
Assert.AreEqual(permissionProperties.PermissionMode, permission.PermissionMode);
//delete resource with PermissionMode.Read
using (CosmosClient tokenCosmosClient = TestCommon.CreateCosmosClient(clientOptions: null, resourceToken: permission.Token))
{
Container tokenContainer = tokenCosmosClient.GetContainer(this.cosmosDatabase.Id, containerId);
ItemResponse <dynamic> readPermissionItem = await tokenContainer.ReadItemAsync <dynamic>(itemId, partitionKey);
Assert.AreEqual(itemId, readPermissionItem.Resource.id.ToString());
try
{
ItemResponse <dynamic> response = await tokenContainer.DeleteItemAsync <dynamic>(
itemId,
partitionKey);
Assert.Fail();
}
catch (CosmosException ex)
{
Assert.AreEqual(HttpStatusCode.Forbidden, ex.StatusCode);
}
}
//update permission to PermissionMode.All
permissionProperties = new PermissionProperties(permissionId, PermissionMode.All, container);
permissionResponse = await user.GetPermission(permissionId).ReplaceAsync(permissionProperties);
permission = permissionResponse.Resource;
//delete resource with PermissionMode.All
using (CosmosClient tokenCosmosClient = TestCommon.CreateCosmosClient(clientOptions: null, resourceToken: permission.Token))
{
FeedIterator <dynamic> feed = tokenCosmosClient
.GetDatabase(this.cosmosDatabase.Id)
.GetContainer(containerId)
.GetItemQueryIterator <dynamic>(new QueryDefinition("select * from t"));
while (feed.HasMoreResults)
{
FeedResponse <dynamic> response = await feed.ReadNextAsync();
Assert.IsNotNull(response);
}
}
}
private async Task StartDemo()
{
Console.WriteLine("Starting Cosmos DB SQL API Demo!");
//Create a new demo database
string databaseName = "demoDB_" + Guid.NewGuid().ToString().Substring(0, 5);
this.SendMessageToConsoleAndWait($"Creating database {databaseName}...");
this.client = new CosmosClient(EndpointUri, Key);
this.database = await this.client.CreateDatabaseIfNotExistsAsync(databaseName);
//Create a new demo collection inside the demo database.
//This creates a collection with a reserved throughput. You can customize the options using a ContainerProperties object
//This operation has pricing implications.
string containerName = "collection_" + Guid.NewGuid().ToString().Substring(0, 5);
this.SendMessageToConsoleAndWait($"Creating collection demo{containerName}...");
this.container = await this.database.CreateContainerIfNotExistsAsync(containerName, "/LastName");
//Create some documents in the collection
Person person1 = new Person
{
Id = "Person.1",
FirstName = "Santiago",
LastName = "Fernandez",
Devices = new Device[]
{
new Device {
OperatingSystem = "iOS", CameraMegaPixels = 7,
Ram = 16, Usage = "Personal"
},
new Device {
OperatingSystem = "Android", CameraMegaPixels = 12,
Ram = 64, Usage = "Work"
}
},
Gender = "Male",
Address = new Address
{
City = "Seville",
Country = "Spain",
PostalCode = "28973",
Street = "Diagonal",
State = "Andalucia"
},
IsRegistered = true
};
await this.CreateDocumentIfNotExistsAsync(databaseName, containerName, person1);
Person person2 = new Person
{
Id = "Person.2",
FirstName = "Agatha",
LastName = "Smith",
Devices = new Device[]
{
new Device {
OperatingSystem = "iOS", CameraMegaPixels = 12,
Ram = 32, Usage = "Work"
},
new Device {
OperatingSystem = "Windows", CameraMegaPixels = 12,
Ram = 64, Usage = "Personal"
}
},
Gender = "Female",
Address = new Address
{
City = "Laguna Beach",
Country = "United States",
PostalCode = "12345",
Street = "Main",
State = "CA"
},
IsRegistered = true
};
await this.CreateDocumentIfNotExistsAsync(databaseName, containerName, person2);
//Make some queries to the collection
this.SendMessageToConsoleAndWait($"Getting documents from the collection{containerName}...");
//Find documents using LINQ
IQueryable <Person> queryablePeople = this.container.GetItemLinqQueryable <Person>(true)
.Where(p => p.Gender == "Male");
System.Console.WriteLine("Running LINQ query for finding men...");
foreach (Person foundPerson in queryablePeople)
{
System.Console.WriteLine($"\tPerson: {foundPerson}");
}
//Find documents using SQL
var sqlQuery = "SELECT * FROM Person WHERE Person.Gender = 'Female'";
QueryDefinition queryDefinition = new QueryDefinition(sqlQuery);
FeedIterator <Person> peopleResultSetIterator = this.container.GetItemQueryIterator <Person>(queryDefinition);
System.Console.WriteLine("Running SQL query for finding women...");
while (peopleResultSetIterator.HasMoreResults)
{
FeedResponse <Person> currentResultSet = await peopleResultSetIterator.ReadNextAsync();
foreach (Person foundPerson in currentResultSet)
{
System.Console.WriteLine($"\tPerson: {foundPerson}");
}
}
Console.WriteLine("Press any key to continue...");
Console.ReadKey();
//Update documents in a collection
this.SendMessageToConsoleAndWait($"Updating documents in the collection { containerName}...");
person2.FirstName = "Mathew";
person2.Gender = "Male";
await this.container.UpsertItemAsync(person2);
this.SendMessageToConsoleAndWait($"Document modified {person2}");
//Delete a single document from the collection
this.SendMessageToConsoleAndWait($"Deleting documents from the collection { containerName}...");
PartitionKey partitionKey = new PartitionKey(person1.LastName);
await this.container.DeleteItemAsync <Person>(person1.Id, partitionKey);
this.SendMessageToConsoleAndWait($"Document deleted {person1}");
//Delete created demo database and all its children elements
this.SendMessageToConsoleAndWait("Cleaning-up your Cosmos DB account...");
await this.database.DeleteAsync();
}
/// <inheritdoc/>
public async Task <InstanceQueryResponse> GetInstancesFromQuery(
Dictionary <string, StringValues> queryParams,
string continuationToken,
int size)
{
InstanceQueryResponse queryResponse = new InstanceQueryResponse
{
Count = 0,
Instances = new List <Instance>()
};
while (queryResponse.Count < size)
{
FeedOptions feedOptions = new FeedOptions
{
EnableCrossPartitionQuery = true,
MaxItemCount = size - queryResponse.Count,
ResponseContinuationTokenLimitInKb = 7
};
if (!string.IsNullOrEmpty(continuationToken))
{
feedOptions.RequestContinuation = continuationToken;
}
IQueryable <Instance> queryBuilder = _client.CreateDocumentQuery <Instance>(_collectionUri, feedOptions);
try
{
queryBuilder = BuildQueryFromParameters(queryParams, queryBuilder)
.Where(i => !i.Status.IsHardDeleted);
}
catch (Exception e)
{
queryResponse.Exception = e.Message;
return(queryResponse);
}
try
{
IDocumentQuery <Instance> documentQuery = queryBuilder.AsDocumentQuery();
FeedResponse <Instance> feedResponse = await documentQuery.ExecuteNextAsync <Instance>();
if (feedResponse.Count == 0 && !documentQuery.HasMoreResults)
{
queryResponse.ContinuationToken = string.Empty;
break;
}
List <Instance> instances = feedResponse.ToList();
await PostProcess(instances);
queryResponse.Instances.AddRange(instances);
queryResponse.Count += instances.Count;
if (string.IsNullOrEmpty(feedResponse.ResponseContinuation))
{
queryResponse.ContinuationToken = string.Empty;
break;
}
queryResponse.ContinuationToken = feedResponse.ResponseContinuation;
continuationToken = feedResponse.ResponseContinuation;
}
catch (Exception e)
{
_logger.LogError(e, "Exception querying CosmosDB for instances");
queryResponse.Exception = e.Message;
break;
}
}
return(queryResponse);
}
public async Task ParallelizeQueryThroughTokens_OfT()
{
ContainerInternal container = null;
try
{
// Create a container large enough to have at least 2 partitions
ContainerResponse containerResponse = await this.database.CreateContainerAsync(
id : Guid.NewGuid().ToString(),
partitionKeyPath : "/id",
throughput : 15000);
container = (ContainerInlineCore)containerResponse;
List <string> generatedIds = Enumerable.Range(0, 1000).Select(n => $"BasicItem{n}").ToList();
foreach (string id in generatedIds)
{
string item = $@"
{{
""id"": ""{id}""
}}";
using (ResponseMessage createResponse = await container.CreateItemStreamAsync(
QueryFeedRangeTests.GenerateStreamFromString(item),
new Cosmos.PartitionKey(id)))
{
Assert.IsTrue(createResponse.IsSuccessStatusCode);
}
}
IReadOnlyList <FeedRange> feedTokens = await container.GetFeedRangesAsync();
Assert.IsTrue(feedTokens.Count > 1, " RUs of the container needs to be increased to ensure at least 2 partitions.");
List <Task <List <string> > > tasks = feedTokens.Select(async feedToken =>
{
List <string> results = new List <string>();
FeedIterator <ToDoActivity> feedIterator = container.GetItemQueryIterator <ToDoActivity>(queryDefinition: new QueryDefinition("select * from T where STARTSWITH(T.id, \"BasicItem\")"), feedRange: feedToken, requestOptions: new QueryRequestOptions()
{
MaxItemCount = 10
});
string continuation = null;
while (feedIterator.HasMoreResults)
{
FeedResponse <ToDoActivity> response = await feedIterator.ReadNextAsync();
foreach (ToDoActivity toDoActivity in response)
{
results.Add(toDoActivity.id);
}
continuation = response.ContinuationToken;
break;
}
feedIterator = container.GetItemQueryIterator <ToDoActivity>(queryDefinition: new QueryDefinition("select * from T where STARTSWITH(T.id, \"BasicItem\")"), feedRange: feedToken, continuationToken: continuation, requestOptions: new QueryRequestOptions()
{
MaxItemCount = 10
});
while (feedIterator.HasMoreResults)
{
FeedResponse <ToDoActivity> response = await feedIterator.ReadNextAsync();
foreach (ToDoActivity toDoActivity in response)
{
results.Add(toDoActivity.id);
}
}
return(results);
}).ToList();
await Task.WhenAll(tasks);
CollectionAssert.AreEquivalent(generatedIds, tasks.SelectMany(t => t.Result).ToList());
}
finally
{
await container?.DeleteContainerAsync();
}
}
public static async void Run([CosmosDBTrigger(
databaseName: "contoso",
collectionName: "orders-aggregate",
ConnectionStringSetting = "CosmosConnectionString",
LeaseCollectionName = "leases")] IReadOnlyList <Microsoft.Azure.Documents.Document> input, ILogger log)
{
if (input != null && input.Count > 0)
{
log.LogInformation("Documents modified " + input.Count);
log.LogInformation("First document Id " + input[0].Id);
string connStr = Environment.GetEnvironmentVariable("CosmosConnectionString");
CosmosClient cosmosClient = new CosmosClient(connStr);
Microsoft.Azure.Cosmos.Database db = cosmosClient.GetDatabase("contoso");
Container container = db.GetContainer("orders-for-hour");
// custQuery is an IEnumerable<IGrouping<string, Customer>>
var dateQuery =
from o in input
group o by o.GetPropertyValue <string>("OrderDateHour") into orderDateGroup
select orderDateGroup;
log.LogInformation(dateQuery.Count().ToString());
foreach (var dq in dateQuery)
{
log.LogInformation("DateQuery: " + dq.Key);
var sqlQueryText = $"SELECT * FROM c WHERE c.id = '{dq.Key}'";
log.LogInformation("Running query: {0}\n", sqlQueryText);
QueryDefinition queryDefinition = new QueryDefinition(sqlQueryText);
FeedIterator <OrderEvent> queryResultSetIterator = container.GetItemQueryIterator <OrderEvent>(queryDefinition);
List <OrderEvent> orders = new List <OrderEvent>();
while (queryResultSetIterator.HasMoreResults)
{
log.LogInformation("In loop");
FeedResponse <OrderEvent> currentResultSet = await queryResultSetIterator.ReadNextAsync();
foreach (OrderEvent order in currentResultSet)
{
orders.Add(order);
log.LogInformation("\tRead {0}\n", order);
}
}
log.LogInformation("Orders Count: " + orders.Count);
OrderEvent currentOrder = new OrderEvent()
{
id = dq.Key, Count = dq.Count(), Total = dq.Sum(x => x.GetPropertyValue <decimal>("Total"))
};
if (orders.Count == 0)
{
log.LogInformation("Addin entry");
try
{
ItemResponse <OrderEvent> orderResponse = await container.CreateItemAsync <OrderEvent>(currentOrder);
}
catch (Exception ex)
{
log.LogInformation("Exception: " + ex.Message);
}
}
else
{
ItemResponse <OrderEvent> orderResponse = await container.ReadItemAsync <OrderEvent>(dq.Key, new PartitionKey(dq.Key));
int currentCount = orderResponse.Resource.Count;
decimal currentTotal = orderResponse.Resource.Total;
OrderEvent newOrder = new OrderEvent()
{
id = dq.Key, Count = currentCount + currentOrder.Count, Total = currentTotal + currentOrder.Total
};
orderResponse = await container.ReplaceItemAsync <OrderEvent>(newOrder, newOrder.id);
}
}
}
}
private async Task ValidateServerSideQueryEvalWithPaginationScenario()
{
DocumentClient client = TestCommon.CreateClient(false, defaultConsistencyLevel: ConsistencyLevel.Session);
CosmosDatabaseSettings database = TestCommon.CreateOrGetDatabase(client);
CosmosContainerSettings collection = new CosmosContainerSettings
{
Id = "ConsistentCollection"
};
collection.IndexingPolicy.IndexingMode = IndexingMode.Consistent;
collection = client.Create <CosmosContainerSettings>(
database.ResourceId,
collection);
//Do script post to insert as many document as we could in a tight loop.
string script = @"function() {
var output = 0;
var client = getContext().getCollection();
function callback(err, docCreated) {
if(err) throw 'Error while creating document';
output++;
getContext().getResponse().setBody(output);
if(output < 50)
client.createDocument(client.getSelfLink(), { id: 'testDoc' + output, title : 'My Book'}, {}, callback);
};
client.createDocument(client.getSelfLink(), { id: 'testDoc' + output, title : 'My Book'}, {}, callback); }";
StoredProcedureResponse <int> scriptResponse = null;
int totalNumberOfDocuments = GatewayTests.CreateExecuteAndDeleteProcedure(client, collection, script, out scriptResponse);
int pageSize = 5;
int totalHit = 0;
IDocumentQuery <Book> documentQuery =
(from book in client.CreateDocumentQuery <Book>(
collection.SelfLink, new FeedOptions {
MaxItemCount = pageSize
})
where book.Title == "My Book"
select book).AsDocumentQuery();
while (documentQuery.HasMoreResults)
{
FeedResponse <dynamic> pagedResult = await documentQuery.ExecuteNextAsync();
string isUnfiltered = pagedResult.ResponseHeaders[HttpConstants.HttpHeaders.IsFeedUnfiltered];
Assert.IsTrue(string.IsNullOrEmpty(isUnfiltered), "Query is evaulated in client");
Assert.IsTrue(pagedResult.Count <= pageSize, "Page size is not honored in client site eval");
if (totalHit != 0 && documentQuery.HasMoreResults)
{
//Except first page and last page we should have seen client continuation token.
Assert.IsFalse(pagedResult.ResponseHeaders[HttpConstants.HttpHeaders.Continuation].Contains(HttpConstants.Delimiters.ClientContinuationDelimiter),
"Client continuation is missing from the response continuation");
}
totalHit += pagedResult.Count;
}
Assert.AreEqual(totalHit, totalNumberOfDocuments, "Didnt get all the documents");
//Do with default pagination.
documentQuery =
(from book in client.CreateDocumentQuery <Book>(
collection.SelfLink)
where book.Title == "My Book"
select book).AsDocumentQuery();
totalHit = 0;
while (documentQuery.HasMoreResults)
{
FeedResponse <dynamic> pagedResult = await documentQuery.ExecuteNextAsync();
string isUnfiltered = pagedResult.ResponseHeaders[HttpConstants.HttpHeaders.IsFeedUnfiltered];
Assert.IsTrue(string.IsNullOrEmpty(isUnfiltered), "Query is evaulated in client");
Assert.IsTrue(pagedResult.Count == totalNumberOfDocuments, "Page size is not honored in client site eval");
totalHit += pagedResult.Count;
}
Assert.AreEqual(totalHit, totalNumberOfDocuments, "Didnt get all the documents");
}
