private async Task <IReadOnlyList <DocumentServiceLeaseCore> > ListDocumentsAsync(string prefix)
{
if (string.IsNullOrEmpty(prefix))
{
throw new ArgumentException("Prefix must be non-empty string", nameof(prefix));
}
using FeedIterator iterator = this.container.GetItemQueryStreamIterator(
"SELECT * FROM c WHERE STARTSWITH(c.id, '" + prefix + "')",
continuationToken: null,
requestOptions: queryRequestOptions);
List <DocumentServiceLeaseCore> leases = new List <DocumentServiceLeaseCore>();
while (iterator.HasMoreResults)
{
using (ResponseMessage responseMessage = await iterator.ReadNextAsync().ConfigureAwait(false))
{
responseMessage.EnsureSuccessStatusCode();
leases.AddRange(CosmosFeedResponseSerializer.FromFeedResponseStream <DocumentServiceLeaseCore>(
CosmosContainerExtensions.DefaultJsonSerializer,
responseMessage.Content));
}
}
return(leases);
}
private Task DispatchChangesAsync(ResponseMessage response, CancellationToken cancellationToken)
{
ChangeFeedObserverContext context = new ChangeFeedObserverContextCore <T>(this.options.LeaseToken, response, this.checkpointer);
IEnumerable <T> asFeedResponse;
try
{
asFeedResponse = CosmosFeedResponseSerializer.FromFeedResponseStream <T>(
this.serializerCore,
response.Content);
}
catch (Exception serializationException)
{
// Error using custom serializer to parse stream
throw new ObserverException(serializationException);
}
// When StartFromBeginning is used, the first request returns OK but no content
if (!asFeedResponse.Any())
{
return(Task.CompletedTask);
}
List <T> asReadOnlyList = new List <T>(asFeedResponse);
return(this.observer.ProcessChangesAsync(context, asReadOnlyList, cancellationToken));
}
internal static ReadFeedResponse <TInput> CreateResponse <TInput>(
ResponseMessage responseMessage,
CosmosSerializerCore serializerCore)
{
using (responseMessage)
{
// ReadFeed can return 304 on Change Feed responses
if (responseMessage.StatusCode != HttpStatusCode.NotModified)
{
responseMessage.EnsureSuccessStatusCode();
}
IReadOnlyCollection <TInput> resources = CosmosFeedResponseSerializer.FromFeedResponseStream <TInput>(
serializerCore,
responseMessage.Content);
ReadFeedResponse <TInput> readFeedResponse = new ReadFeedResponse <TInput>(
httpStatusCode: responseMessage.StatusCode,
resources: resources,
resourceCount: resources.Count,
responseMessageHeaders: responseMessage.Headers,
diagnostics: responseMessage.Diagnostics);
return(readFeedResponse);
}
}
public async Task TestWithFixedLeaseContainer()
{
await NonPartitionedContainerHelper.CreateNonPartitionedContainer(
this.database,
"fixedLeases");
Container fixedLeasesContainer = this.cosmosClient.GetContainer(this.database.Id, "fixedLeases");
try
{
int partitionKey = 0;
ManualResetEvent allDocsProcessed = new ManualResetEvent(false);
int processedDocCount = 0;
string accumulator = string.Empty;
ChangeFeedProcessor processor = this.Container
.GetChangeFeedProcessorBuilder("test", (ChangeFeedProcessorContext context, Stream stream, CancellationToken token) =>
{
this.ValidateContext(context);
IEnumerable <JObject> asEnumerable = CosmosFeedResponseSerializer.FromFeedResponseStream <JObject>(this.serializerCore, stream);
processedDocCount += asEnumerable.Count();
foreach (JObject doc in asEnumerable)
{
accumulator += doc["id"].ToString() + ".";
}
if (processedDocCount == 10)
{
allDocsProcessed.Set();
}
return(Task.CompletedTask);
})
.WithInstanceName("random")
.WithLeaseContainer(fixedLeasesContainer).Build();
// Start the processor, insert 1 document to generate a checkpoint
await processor.StartAsync();
await Task.Delay(BaseChangeFeedClientHelper.ChangeFeedSetupTime);
foreach (int id in Enumerable.Range(0, 10))
{
await this.Container.CreateItemAsync <dynamic>(new { id = id.ToString(), pk = partitionKey });
}
bool isStartOk = allDocsProcessed.WaitOne(10 * BaseChangeFeedClientHelper.ChangeFeedSetupTime);
await processor.StopAsync();
Assert.IsTrue(isStartOk, "Timed out waiting for docs to process");
Assert.AreEqual("0.1.2.3.4.5.6.7.8.9.", accumulator);
}
finally
{
await fixedLeasesContainer.DeleteContainerAsync();
}
}
public async Task TestWithRunningProcessor_WithManualCheckpoint()
{
int partitionKey = 0;
ManualResetEvent allDocsProcessed = new ManualResetEvent(false);
int processedDocCount = 0;
string accumulator = string.Empty;
ChangeFeedProcessor processor = this.Container
.GetChangeFeedProcessorBuilderWithManualCheckpoint("test", async(ChangeFeedProcessorContext context, Stream stream, Func <Task> checkpointAsync, CancellationToken token) =>
{
this.ValidateContext(context);
IEnumerable <JObject> docs = CosmosFeedResponseSerializer.FromFeedResponseStream <JObject>(this.serializerCore, stream);
processedDocCount += docs.Count();
foreach (dynamic doc in docs)
{
accumulator += doc.id.ToString() + ".";
}
if (processedDocCount == 3)
{
// Throwing on the 3rd document, since we checkpointed only on the 1st, we would repeat 2nd and 3rd
throw new Exception("Stop here");
}
if (processedDocCount == 1)
{
// Checkpointing on the first document to be able to have a point to rollback to
await checkpointAsync();
}
if (processedDocCount == 12)
{
allDocsProcessed.Set();
}
})
.WithInstanceName("random")
.WithMaxItems(1)
.WithLeaseContainer(this.LeaseContainer).Build();
// Start the processor, insert 1 document to generate a checkpoint
await processor.StartAsync();
await Task.Delay(BaseChangeFeedClientHelper.ChangeFeedSetupTime);
foreach (int id in Enumerable.Range(0, 10))
{
await this.Container.CreateItemAsync <dynamic>(new { id = id.ToString(), pk = partitionKey });
}
bool isStartOk = allDocsProcessed.WaitOne(30 * BaseChangeFeedClientHelper.ChangeFeedSetupTime);
await processor.StopAsync();
Assert.IsTrue(isStartOk, "Timed out waiting for docs to process");
Assert.AreEqual("0.1.2.1.2.3.4.5.6.7.8.9.", accumulator);
}
private static IEnumerable <JObject> GetItemsFromResponse(ResponseMessage response)
{
if (response.Content == null)
{
return(new Collection <JObject>());
}
return(CosmosFeedResponseSerializer.FromFeedResponseStream <JObject>(
CosmosContainerExtensions.DefaultJsonSerializer,
response.Content));
}
public void ByteParsingToFindJsonArrayWithSeriliazation()
{
using (MemoryStream ms = new MemoryStream(this.payloadBytes))
{
IReadOnlyCollection <ToDoActivity> results = CosmosFeedResponseSerializer.FromFeedResponseStream <ToDoActivity>(
this.serializerCore,
ms);
if (results.Count != 1000)
{
throw new Exception();
}
}
}
public void ByteParsingToFindJsonArray()
{
using (MemoryStream ms = new MemoryStream(this.payloadBytes))
{
long length = ms.Length;
using (MemoryStream memoryStream = CosmosFeedResponseSerializer.GetStreamWithoutServiceEnvelope(ms))
{
if (length == memoryStream.Length)
{
throw new Exception();
}
}
}
}
public async Task CosmosConflictsStreamIteratorBuildsSettings()
{
string conflictResponsePayload = @"{ ""Conflicts"":[{
""id"": ""Conflict1"",
""operationType"": ""Replace"",
""resourceType"": ""trigger""
}]}";
JObject jObject = JObject.Parse(conflictResponsePayload);
using CosmosClient mockClient = MockCosmosUtil.CreateMockCosmosClient(
(cosmosClientBuilder) => cosmosClientBuilder.WithConnectionModeDirect());
Container container = mockClient.GetContainer("database", "container");
FeedIterator feedIterator = container.Conflicts.GetConflictQueryStreamIterator();
TestHandler testHandler = new TestHandler((request, cancellationToken) =>
{
Assert.AreEqual(OperationType.ReadFeed, request.OperationType);
Assert.AreEqual(ResourceType.Conflict, request.ResourceType);
ResponseMessage handlerResponse = TestHandler.ReturnSuccess().Result;
MemoryStream stream = new MemoryStream();
StreamWriter writer = new StreamWriter(stream);
writer.Write(conflictResponsePayload);
writer.Flush();
stream.Position = 0;
handlerResponse.Content = stream;
return(Task.FromResult(handlerResponse));
});
mockClient.RequestHandler.InnerHandler = testHandler;
ResponseMessage streamResponse = await feedIterator.ReadNextAsync();
IEnumerable <ConflictProperties> response = CosmosFeedResponseSerializer.FromFeedResponseStream <ConflictProperties>(
MockCosmosUtil.Serializer,
streamResponse.Content);
Assert.AreEqual(1, response.Count());
ConflictProperties responseSettings = response.FirstOrDefault();
Assert.IsNotNull(responseSettings);
Assert.AreEqual("Conflict1", responseSettings.Id);
Assert.AreEqual(Cosmos.OperationKind.Replace, responseSettings.OperationKind);
Assert.AreEqual(typeof(TriggerProperties), responseSettings.ResourceType);
}
public async Task TestWithStartTime_Beginning()
{
int partitionKey = 0;
ManualResetEvent allDocsProcessed = new ManualResetEvent(false);
int processedDocCount = 0;
string accumulator = string.Empty;
ChangeFeedProcessor processor = this.Container
.GetChangeFeedProcessorBuilder("test", (ChangeFeedProcessorContext context, Stream stream, CancellationToken token) =>
{
this.ValidateContext(context);
IEnumerable <JObject> asEnumerable = CosmosFeedResponseSerializer.FromFeedResponseStream <JObject>(this.serializerCore, stream);
processedDocCount += asEnumerable.Count();
foreach (JObject doc in asEnumerable)
{
accumulator += doc["id"].ToString() + ".";
}
if (processedDocCount == 5)
{
allDocsProcessed.Set();
}
return(Task.CompletedTask);
})
.WithStartTime(DateTime.MinValue.ToUniversalTime())
.WithInstanceName("random")
.WithLeaseContainer(this.LeaseContainer).Build();
await processor.StartAsync();
await Task.Delay(BaseChangeFeedClientHelper.ChangeFeedSetupTime);
foreach (int id in Enumerable.Range(0, 5))
{
await this.Container.CreateItemAsync <dynamic>(new { id = $"doc{id}", pk = partitionKey });
}
// Letting processor initialize and pickup changes
bool isStartOk = allDocsProcessed.WaitOne(10 * BaseChangeFeedClientHelper.ChangeFeedSetupTime);
await processor.StopAsync();
Assert.IsTrue(isStartOk, "Timed out waiting for docs to process");
Assert.AreEqual("doc0.doc1.doc2.doc3.doc4.", accumulator);
}
public async Task ReadManyStreamTest()
{
List <(string, PartitionKey)> itemList = new List <(string, PartitionKey)>();
for (int i = 0; i < 5; i++)
{
itemList.Add((i.ToString(), new PartitionKey("pk" + i.ToString())));
}
using (ResponseMessage responseMessage = await this.Container.ReadManyItemsStreamAsync(itemList))
{
Assert.IsNotNull(responseMessage);
Assert.IsTrue(responseMessage.Headers.RequestCharge > 0);
Assert.IsNotNull(responseMessage.Diagnostics);
ToDoActivity[] items = this.cosmosClient.ClientContext.SerializerCore.FromFeedStream <ToDoActivity>(
CosmosFeedResponseSerializer.GetStreamWithoutServiceEnvelope(responseMessage.Content));
Assert.AreEqual(items.Length, 5);
}
}
public async Task ReadManyWithIdasPk()
{
string PartitionKey = "/id";
ContainerProperties containerSettings = new ContainerProperties(id: Guid.NewGuid().ToString(), partitionKeyPath: PartitionKey);
Container container = await this.database.CreateContainerAsync(containerSettings);
List <(string, PartitionKey)> itemList = new List <(string, PartitionKey)>();
for (int i = 0; i < 5; i++)
{
itemList.Add((i.ToString(), new PartitionKey(i.ToString())));
}
// Create items with different pk values
for (int i = 0; i < 5; i++)
{
ToDoActivity item = ToDoActivity.CreateRandomToDoActivity();
item.id = i.ToString();
ItemResponse <ToDoActivity> itemResponse = await container.CreateItemAsync(item);
Assert.AreEqual(HttpStatusCode.Created, itemResponse.StatusCode);
}
using (ResponseMessage responseMessage = await container.ReadManyItemsStreamAsync(itemList))
{
Assert.IsNotNull(responseMessage);
Assert.IsTrue(responseMessage.Headers.RequestCharge > 0);
Assert.IsNotNull(responseMessage.Diagnostics);
ToDoActivity[] items = this.cosmosClient.ClientContext.SerializerCore.FromFeedStream <ToDoActivity>(
CosmosFeedResponseSerializer.GetStreamWithoutServiceEnvelope(responseMessage.Content));
Assert.AreEqual(items.Length, 5);
}
FeedResponse <ToDoActivity> feedResponse = await container.ReadManyItemsAsync <ToDoActivity>(itemList);
Assert.IsNotNull(feedResponse);
Assert.AreEqual(feedResponse.Count, 5);
Assert.IsTrue(feedResponse.Headers.RequestCharge > 0);
Assert.IsNotNull(feedResponse.Diagnostics);
}
internal static ReadFeedResponse <TInput> CreateResponse <TInput>(
ResponseMessage responseMessage,
CosmosSerializerCore serializerCore)
{
using (responseMessage)
{
responseMessage.EnsureSuccessStatusCode();
IReadOnlyCollection <TInput> resources = CosmosFeedResponseSerializer.FromFeedResponseStream <TInput>(
serializerCore,
responseMessage.Content);
ReadFeedResponse <TInput> readFeedResponse = new ReadFeedResponse <TInput>(
httpStatusCode: responseMessage.StatusCode,
resources: resources,
responseMessageHeaders: responseMessage.Headers,
diagnostics: responseMessage.Diagnostics);
return(readFeedResponse);
}
}
public async Task TestWithRunningProcessor_WithManualCheckpoint()
{
int partitionKey = 0;
ManualResetEvent allDocsProcessed = new ManualResetEvent(false);
int processedDocCount = 0;
string accumulator = string.Empty;
ChangeFeedProcessor processor = this.Container
.GetChangeFeedProcessorBuilderWithManualCheckpoint("test", async(ChangeFeedProcessorContext context, Stream stream, Func <Task <(bool isSuccess, Exception error)> > tryCheckpointAsync, CancellationToken token) =>
{
this.ValidateContext(context);
IEnumerable <JObject> docs = CosmosFeedResponseSerializer.FromFeedResponseStream <JObject>(this.serializerCore, stream);
processedDocCount += docs.Count();
foreach (dynamic doc in docs)
{
accumulator += doc.id.ToString() + ".";
}
if (processedDocCount == 3)
{
// Throwing on the 3rd document, since we checkpointed only on the 1st, we would repeat 2nd and 3rd
throw new Exception("Stop here");
}
if (processedDocCount == 1)
{
// Checkpointing on the first document to be able to have a point to rollback to
(bool isSuccess, Exception exception) = await tryCheckpointAsync();
Assert.IsTrue(isSuccess);
Assert.IsNull(exception);
}
if (processedDocCount == 12)
{
allDocsProcessed.Set();
}
})
internal static MemoryStream GetStreamWithoutServiceEnvelope(
Stream streamWithServiceEnvelope)
{
using (streamWithServiceEnvelope)
{
ReadOnlyMemory <byte> content;
if (!(streamWithServiceEnvelope is MemoryStream memoryStreamWithEnvelope))
{
memoryStreamWithEnvelope = new MemoryStream();
streamWithServiceEnvelope.CopyTo(memoryStreamWithEnvelope);
memoryStreamWithEnvelope.Position = 0;
}
if (memoryStreamWithEnvelope.TryGetBuffer(out ArraySegment <byte> buffer))
{
content = buffer;
}
else
{
content = memoryStreamWithEnvelope.ToArray();
}
int start = CosmosFeedResponseSerializer.GetArrayStartPosition(content);
int end = CosmosFeedResponseSerializer.GetArrayEndPosition(content);
ReadOnlyMemory <byte> spanwithOnlyArray = content.Slice(
start: start,
length: end - start + 1);
if (!MemoryMarshal.TryGetArray(spanwithOnlyArray, out ArraySegment <byte> resultAsArray))
{
resultAsArray = new ArraySegment <byte>(spanwithOnlyArray.ToArray());
}
MemoryStream arrayOnlyStream = new MemoryStream(resultAsArray.Array, resultAsArray.Offset, resultAsArray.Count, writable: false, publiclyVisible: true);
return(arrayOnlyStream);
}
}
private bool ValidateStream(Stream stream)
{
IEnumerable <MyDocument> asEnumerable = CosmosFeedResponseSerializer.FromFeedResponseStream <MyDocument>(this.serializerCore, stream);
return(this.documents.SequenceEqual(asEnumerable, new MyDocument.Comparer()));
}
public async Task TestReducePageSizeScenario()
{
int partitionKey = 0;
// Create some docs to make sure that one separate response is returned for 1st execute of query before retries.
// These are to make sure continuation token is passed along during retries.
string sprocId = "createTwoDocs";
string sprocBody = @"function(startIndex) { for (var i = 0; i < 2; ++i) __.createDocument(
__.getSelfLink(),
{ id: 'doc' + (i + startIndex).toString(), value: 'y'.repeat(1500000), pk:0 },
err => { if (err) throw err;}
);}";
Scripts scripts = this.Container.Scripts;
StoredProcedureResponse storedProcedureResponse =
await scripts.CreateStoredProcedureAsync(new StoredProcedureProperties(sprocId, sprocBody));
ManualResetEvent allDocsProcessed = new ManualResetEvent(false);
int processedDocCount = 0;
string accumulator = string.Empty;
ChangeFeedProcessor processor = this.Container
.GetChangeFeedProcessorBuilder("test", (ChangeFeedProcessorContext context, Stream stream, CancellationToken token) =>
{
this.ValidateContext(context);
IEnumerable <JObject> asEnumerable = CosmosFeedResponseSerializer.FromFeedResponseStream <JObject>(this.serializerCore, stream);
processedDocCount += asEnumerable.Count();
foreach (JObject doc in asEnumerable)
{
accumulator += doc["id"].ToString() + ".";
}
if (processedDocCount == 5)
{
allDocsProcessed.Set();
}
return(Task.CompletedTask);
})
.WithStartFromBeginning()
.WithInstanceName("random")
.WithMaxItems(6)
.WithLeaseContainer(this.LeaseContainer).Build();
// Generate the payload
await scripts.ExecuteStoredProcedureAsync <object>(
sprocId,
new PartitionKey(partitionKey),
new dynamic[] { 0 });
// Create 3 docs each 1.5MB. All 3 do not fit into MAX_RESPONSE_SIZE (4 MB). 2nd and 3rd are in same transaction.
string content = string.Format("{{\"id\": \"doc2\", \"value\": \"{0}\", \"pk\": 0}}", new string('x', 1500000));
await this.Container.CreateItemAsync(JsonConvert.DeserializeObject <dynamic>(content), new PartitionKey(partitionKey));
await scripts.ExecuteStoredProcedureAsync <object>(sprocId, new PartitionKey(partitionKey), new dynamic[] { 3 });
await processor.StartAsync();
// Letting processor initialize and pickup changes
bool isStartOk = allDocsProcessed.WaitOne(10 * BaseChangeFeedClientHelper.ChangeFeedSetupTime);
await processor.StopAsync();
Assert.IsTrue(isStartOk, "Timed out waiting for docs to process");
Assert.AreEqual("doc0.doc1.doc2.doc3.doc4.", accumulator);
}