private async Task <IReadOnlyList <DocumentServiceLeaseCore> > ListDocumentsAsync(string prefix) { if (string.IsNullOrEmpty(prefix)) { throw new ArgumentException("Prefix must be non-empty string", nameof(prefix)); } using FeedIterator iterator = this.container.GetItemQueryStreamIterator( "SELECT * FROM c WHERE STARTSWITH(c.id, '" + prefix + "')", continuationToken: null, requestOptions: queryRequestOptions); List <DocumentServiceLeaseCore> leases = new List <DocumentServiceLeaseCore>(); while (iterator.HasMoreResults) { using (ResponseMessage responseMessage = await iterator.ReadNextAsync().ConfigureAwait(false)) { responseMessage.EnsureSuccessStatusCode(); leases.AddRange(CosmosFeedResponseSerializer.FromFeedResponseStream <DocumentServiceLeaseCore>( CosmosContainerExtensions.DefaultJsonSerializer, responseMessage.Content)); } } return(leases); }
private Task DispatchChangesAsync(ResponseMessage response, CancellationToken cancellationToken) { ChangeFeedObserverContext context = new ChangeFeedObserverContextCore <T>(this.options.LeaseToken, response, this.checkpointer); IEnumerable <T> asFeedResponse; try { asFeedResponse = CosmosFeedResponseSerializer.FromFeedResponseStream <T>( this.serializerCore, response.Content); } catch (Exception serializationException) { // Error using custom serializer to parse stream throw new ObserverException(serializationException); } // When StartFromBeginning is used, the first request returns OK but no content if (!asFeedResponse.Any()) { return(Task.CompletedTask); } List <T> asReadOnlyList = new List <T>(asFeedResponse); return(this.observer.ProcessChangesAsync(context, asReadOnlyList, cancellationToken)); }
internal static ReadFeedResponse <TInput> CreateResponse <TInput>( ResponseMessage responseMessage, CosmosSerializerCore serializerCore) { using (responseMessage) { // ReadFeed can return 304 on Change Feed responses if (responseMessage.StatusCode != HttpStatusCode.NotModified) { responseMessage.EnsureSuccessStatusCode(); } IReadOnlyCollection <TInput> resources = CosmosFeedResponseSerializer.FromFeedResponseStream <TInput>( serializerCore, responseMessage.Content); ReadFeedResponse <TInput> readFeedResponse = new ReadFeedResponse <TInput>( httpStatusCode: responseMessage.StatusCode, resources: resources, resourceCount: resources.Count, responseMessageHeaders: responseMessage.Headers, diagnostics: responseMessage.Diagnostics); return(readFeedResponse); } }
public async Task TestWithFixedLeaseContainer() { await NonPartitionedContainerHelper.CreateNonPartitionedContainer( this.database, "fixedLeases"); Container fixedLeasesContainer = this.cosmosClient.GetContainer(this.database.Id, "fixedLeases"); try { int partitionKey = 0; ManualResetEvent allDocsProcessed = new ManualResetEvent(false); int processedDocCount = 0; string accumulator = string.Empty; ChangeFeedProcessor processor = this.Container .GetChangeFeedProcessorBuilder("test", (ChangeFeedProcessorContext context, Stream stream, CancellationToken token) => { this.ValidateContext(context); IEnumerable <JObject> asEnumerable = CosmosFeedResponseSerializer.FromFeedResponseStream <JObject>(this.serializerCore, stream); processedDocCount += asEnumerable.Count(); foreach (JObject doc in asEnumerable) { accumulator += doc["id"].ToString() + "."; } if (processedDocCount == 10) { allDocsProcessed.Set(); } return(Task.CompletedTask); }) .WithInstanceName("random") .WithLeaseContainer(fixedLeasesContainer).Build(); // Start the processor, insert 1 document to generate a checkpoint await processor.StartAsync(); await Task.Delay(BaseChangeFeedClientHelper.ChangeFeedSetupTime); foreach (int id in Enumerable.Range(0, 10)) { await this.Container.CreateItemAsync <dynamic>(new { id = id.ToString(), pk = partitionKey }); } bool isStartOk = allDocsProcessed.WaitOne(10 * BaseChangeFeedClientHelper.ChangeFeedSetupTime); await processor.StopAsync(); Assert.IsTrue(isStartOk, "Timed out waiting for docs to process"); Assert.AreEqual("0.1.2.3.4.5.6.7.8.9.", accumulator); } finally { await fixedLeasesContainer.DeleteContainerAsync(); } }
public async Task TestWithRunningProcessor_WithManualCheckpoint() { int partitionKey = 0; ManualResetEvent allDocsProcessed = new ManualResetEvent(false); int processedDocCount = 0; string accumulator = string.Empty; ChangeFeedProcessor processor = this.Container .GetChangeFeedProcessorBuilderWithManualCheckpoint("test", async(ChangeFeedProcessorContext context, Stream stream, Func <Task> checkpointAsync, CancellationToken token) => { this.ValidateContext(context); IEnumerable <JObject> docs = CosmosFeedResponseSerializer.FromFeedResponseStream <JObject>(this.serializerCore, stream); processedDocCount += docs.Count(); foreach (dynamic doc in docs) { accumulator += doc.id.ToString() + "."; } if (processedDocCount == 3) { // Throwing on the 3rd document, since we checkpointed only on the 1st, we would repeat 2nd and 3rd throw new Exception("Stop here"); } if (processedDocCount == 1) { // Checkpointing on the first document to be able to have a point to rollback to await checkpointAsync(); } if (processedDocCount == 12) { allDocsProcessed.Set(); } }) .WithInstanceName("random") .WithMaxItems(1) .WithLeaseContainer(this.LeaseContainer).Build(); // Start the processor, insert 1 document to generate a checkpoint await processor.StartAsync(); await Task.Delay(BaseChangeFeedClientHelper.ChangeFeedSetupTime); foreach (int id in Enumerable.Range(0, 10)) { await this.Container.CreateItemAsync <dynamic>(new { id = id.ToString(), pk = partitionKey }); } bool isStartOk = allDocsProcessed.WaitOne(30 * BaseChangeFeedClientHelper.ChangeFeedSetupTime); await processor.StopAsync(); Assert.IsTrue(isStartOk, "Timed out waiting for docs to process"); Assert.AreEqual("0.1.2.1.2.3.4.5.6.7.8.9.", accumulator); }
private static IEnumerable <JObject> GetItemsFromResponse(ResponseMessage response) { if (response.Content == null) { return(new Collection <JObject>()); } return(CosmosFeedResponseSerializer.FromFeedResponseStream <JObject>( CosmosContainerExtensions.DefaultJsonSerializer, response.Content)); }
public void ByteParsingToFindJsonArrayWithSeriliazation() { using (MemoryStream ms = new MemoryStream(this.payloadBytes)) { IReadOnlyCollection <ToDoActivity> results = CosmosFeedResponseSerializer.FromFeedResponseStream <ToDoActivity>( this.serializerCore, ms); if (results.Count != 1000) { throw new Exception(); } } }
public void ByteParsingToFindJsonArray() { using (MemoryStream ms = new MemoryStream(this.payloadBytes)) { long length = ms.Length; using (MemoryStream memoryStream = CosmosFeedResponseSerializer.GetStreamWithoutServiceEnvelope(ms)) { if (length == memoryStream.Length) { throw new Exception(); } } } }
public async Task CosmosConflictsStreamIteratorBuildsSettings() { string conflictResponsePayload = @"{ ""Conflicts"":[{ ""id"": ""Conflict1"", ""operationType"": ""Replace"", ""resourceType"": ""trigger"" }]}"; JObject jObject = JObject.Parse(conflictResponsePayload); using CosmosClient mockClient = MockCosmosUtil.CreateMockCosmosClient( (cosmosClientBuilder) => cosmosClientBuilder.WithConnectionModeDirect()); Container container = mockClient.GetContainer("database", "container"); FeedIterator feedIterator = container.Conflicts.GetConflictQueryStreamIterator(); TestHandler testHandler = new TestHandler((request, cancellationToken) => { Assert.AreEqual(OperationType.ReadFeed, request.OperationType); Assert.AreEqual(ResourceType.Conflict, request.ResourceType); ResponseMessage handlerResponse = TestHandler.ReturnSuccess().Result; MemoryStream stream = new MemoryStream(); StreamWriter writer = new StreamWriter(stream); writer.Write(conflictResponsePayload); writer.Flush(); stream.Position = 0; handlerResponse.Content = stream; return(Task.FromResult(handlerResponse)); }); mockClient.RequestHandler.InnerHandler = testHandler; ResponseMessage streamResponse = await feedIterator.ReadNextAsync(); IEnumerable <ConflictProperties> response = CosmosFeedResponseSerializer.FromFeedResponseStream <ConflictProperties>( MockCosmosUtil.Serializer, streamResponse.Content); Assert.AreEqual(1, response.Count()); ConflictProperties responseSettings = response.FirstOrDefault(); Assert.IsNotNull(responseSettings); Assert.AreEqual("Conflict1", responseSettings.Id); Assert.AreEqual(Cosmos.OperationKind.Replace, responseSettings.OperationKind); Assert.AreEqual(typeof(TriggerProperties), responseSettings.ResourceType); }
public async Task TestWithStartTime_Beginning() { int partitionKey = 0; ManualResetEvent allDocsProcessed = new ManualResetEvent(false); int processedDocCount = 0; string accumulator = string.Empty; ChangeFeedProcessor processor = this.Container .GetChangeFeedProcessorBuilder("test", (ChangeFeedProcessorContext context, Stream stream, CancellationToken token) => { this.ValidateContext(context); IEnumerable <JObject> asEnumerable = CosmosFeedResponseSerializer.FromFeedResponseStream <JObject>(this.serializerCore, stream); processedDocCount += asEnumerable.Count(); foreach (JObject doc in asEnumerable) { accumulator += doc["id"].ToString() + "."; } if (processedDocCount == 5) { allDocsProcessed.Set(); } return(Task.CompletedTask); }) .WithStartTime(DateTime.MinValue.ToUniversalTime()) .WithInstanceName("random") .WithLeaseContainer(this.LeaseContainer).Build(); await processor.StartAsync(); await Task.Delay(BaseChangeFeedClientHelper.ChangeFeedSetupTime); foreach (int id in Enumerable.Range(0, 5)) { await this.Container.CreateItemAsync <dynamic>(new { id = $"doc{id}", pk = partitionKey }); } // Letting processor initialize and pickup changes bool isStartOk = allDocsProcessed.WaitOne(10 * BaseChangeFeedClientHelper.ChangeFeedSetupTime); await processor.StopAsync(); Assert.IsTrue(isStartOk, "Timed out waiting for docs to process"); Assert.AreEqual("doc0.doc1.doc2.doc3.doc4.", accumulator); }
public async Task ReadManyStreamTest() { List <(string, PartitionKey)> itemList = new List <(string, PartitionKey)>(); for (int i = 0; i < 5; i++) { itemList.Add((i.ToString(), new PartitionKey("pk" + i.ToString()))); } using (ResponseMessage responseMessage = await this.Container.ReadManyItemsStreamAsync(itemList)) { Assert.IsNotNull(responseMessage); Assert.IsTrue(responseMessage.Headers.RequestCharge > 0); Assert.IsNotNull(responseMessage.Diagnostics); ToDoActivity[] items = this.cosmosClient.ClientContext.SerializerCore.FromFeedStream <ToDoActivity>( CosmosFeedResponseSerializer.GetStreamWithoutServiceEnvelope(responseMessage.Content)); Assert.AreEqual(items.Length, 5); } }
public async Task ReadManyWithIdasPk() { string PartitionKey = "/id"; ContainerProperties containerSettings = new ContainerProperties(id: Guid.NewGuid().ToString(), partitionKeyPath: PartitionKey); Container container = await this.database.CreateContainerAsync(containerSettings); List <(string, PartitionKey)> itemList = new List <(string, PartitionKey)>(); for (int i = 0; i < 5; i++) { itemList.Add((i.ToString(), new PartitionKey(i.ToString()))); } // Create items with different pk values for (int i = 0; i < 5; i++) { ToDoActivity item = ToDoActivity.CreateRandomToDoActivity(); item.id = i.ToString(); ItemResponse <ToDoActivity> itemResponse = await container.CreateItemAsync(item); Assert.AreEqual(HttpStatusCode.Created, itemResponse.StatusCode); } using (ResponseMessage responseMessage = await container.ReadManyItemsStreamAsync(itemList)) { Assert.IsNotNull(responseMessage); Assert.IsTrue(responseMessage.Headers.RequestCharge > 0); Assert.IsNotNull(responseMessage.Diagnostics); ToDoActivity[] items = this.cosmosClient.ClientContext.SerializerCore.FromFeedStream <ToDoActivity>( CosmosFeedResponseSerializer.GetStreamWithoutServiceEnvelope(responseMessage.Content)); Assert.AreEqual(items.Length, 5); } FeedResponse <ToDoActivity> feedResponse = await container.ReadManyItemsAsync <ToDoActivity>(itemList); Assert.IsNotNull(feedResponse); Assert.AreEqual(feedResponse.Count, 5); Assert.IsTrue(feedResponse.Headers.RequestCharge > 0); Assert.IsNotNull(feedResponse.Diagnostics); }
internal static ReadFeedResponse <TInput> CreateResponse <TInput>( ResponseMessage responseMessage, CosmosSerializerCore serializerCore) { using (responseMessage) { responseMessage.EnsureSuccessStatusCode(); IReadOnlyCollection <TInput> resources = CosmosFeedResponseSerializer.FromFeedResponseStream <TInput>( serializerCore, responseMessage.Content); ReadFeedResponse <TInput> readFeedResponse = new ReadFeedResponse <TInput>( httpStatusCode: responseMessage.StatusCode, resources: resources, responseMessageHeaders: responseMessage.Headers, diagnostics: responseMessage.Diagnostics); return(readFeedResponse); } }
public async Task TestWithRunningProcessor_WithManualCheckpoint() { int partitionKey = 0; ManualResetEvent allDocsProcessed = new ManualResetEvent(false); int processedDocCount = 0; string accumulator = string.Empty; ChangeFeedProcessor processor = this.Container .GetChangeFeedProcessorBuilderWithManualCheckpoint("test", async(ChangeFeedProcessorContext context, Stream stream, Func <Task <(bool isSuccess, Exception error)> > tryCheckpointAsync, CancellationToken token) => { this.ValidateContext(context); IEnumerable <JObject> docs = CosmosFeedResponseSerializer.FromFeedResponseStream <JObject>(this.serializerCore, stream); processedDocCount += docs.Count(); foreach (dynamic doc in docs) { accumulator += doc.id.ToString() + "."; } if (processedDocCount == 3) { // Throwing on the 3rd document, since we checkpointed only on the 1st, we would repeat 2nd and 3rd throw new Exception("Stop here"); } if (processedDocCount == 1) { // Checkpointing on the first document to be able to have a point to rollback to (bool isSuccess, Exception exception) = await tryCheckpointAsync(); Assert.IsTrue(isSuccess); Assert.IsNull(exception); } if (processedDocCount == 12) { allDocsProcessed.Set(); } })
internal static MemoryStream GetStreamWithoutServiceEnvelope( Stream streamWithServiceEnvelope) { using (streamWithServiceEnvelope) { ReadOnlyMemory <byte> content; if (!(streamWithServiceEnvelope is MemoryStream memoryStreamWithEnvelope)) { memoryStreamWithEnvelope = new MemoryStream(); streamWithServiceEnvelope.CopyTo(memoryStreamWithEnvelope); memoryStreamWithEnvelope.Position = 0; } if (memoryStreamWithEnvelope.TryGetBuffer(out ArraySegment <byte> buffer)) { content = buffer; } else { content = memoryStreamWithEnvelope.ToArray(); } int start = CosmosFeedResponseSerializer.GetArrayStartPosition(content); int end = CosmosFeedResponseSerializer.GetArrayEndPosition(content); ReadOnlyMemory <byte> spanwithOnlyArray = content.Slice( start: start, length: end - start + 1); if (!MemoryMarshal.TryGetArray(spanwithOnlyArray, out ArraySegment <byte> resultAsArray)) { resultAsArray = new ArraySegment <byte>(spanwithOnlyArray.ToArray()); } MemoryStream arrayOnlyStream = new MemoryStream(resultAsArray.Array, resultAsArray.Offset, resultAsArray.Count, writable: false, publiclyVisible: true); return(arrayOnlyStream); } }
private bool ValidateStream(Stream stream) { IEnumerable <MyDocument> asEnumerable = CosmosFeedResponseSerializer.FromFeedResponseStream <MyDocument>(this.serializerCore, stream); return(this.documents.SequenceEqual(asEnumerable, new MyDocument.Comparer())); }
public async Task TestReducePageSizeScenario() { int partitionKey = 0; // Create some docs to make sure that one separate response is returned for 1st execute of query before retries. // These are to make sure continuation token is passed along during retries. string sprocId = "createTwoDocs"; string sprocBody = @"function(startIndex) { for (var i = 0; i < 2; ++i) __.createDocument( __.getSelfLink(), { id: 'doc' + (i + startIndex).toString(), value: 'y'.repeat(1500000), pk:0 }, err => { if (err) throw err;} );}"; Scripts scripts = this.Container.Scripts; StoredProcedureResponse storedProcedureResponse = await scripts.CreateStoredProcedureAsync(new StoredProcedureProperties(sprocId, sprocBody)); ManualResetEvent allDocsProcessed = new ManualResetEvent(false); int processedDocCount = 0; string accumulator = string.Empty; ChangeFeedProcessor processor = this.Container .GetChangeFeedProcessorBuilder("test", (ChangeFeedProcessorContext context, Stream stream, CancellationToken token) => { this.ValidateContext(context); IEnumerable <JObject> asEnumerable = CosmosFeedResponseSerializer.FromFeedResponseStream <JObject>(this.serializerCore, stream); processedDocCount += asEnumerable.Count(); foreach (JObject doc in asEnumerable) { accumulator += doc["id"].ToString() + "."; } if (processedDocCount == 5) { allDocsProcessed.Set(); } return(Task.CompletedTask); }) .WithStartFromBeginning() .WithInstanceName("random") .WithMaxItems(6) .WithLeaseContainer(this.LeaseContainer).Build(); // Generate the payload await scripts.ExecuteStoredProcedureAsync <object>( sprocId, new PartitionKey(partitionKey), new dynamic[] { 0 }); // Create 3 docs each 1.5MB. All 3 do not fit into MAX_RESPONSE_SIZE (4 MB). 2nd and 3rd are in same transaction. string content = string.Format("{{\"id\": \"doc2\", \"value\": \"{0}\", \"pk\": 0}}", new string('x', 1500000)); await this.Container.CreateItemAsync(JsonConvert.DeserializeObject <dynamic>(content), new PartitionKey(partitionKey)); await scripts.ExecuteStoredProcedureAsync <object>(sprocId, new PartitionKey(partitionKey), new dynamic[] { 3 }); await processor.StartAsync(); // Letting processor initialize and pickup changes bool isStartOk = allDocsProcessed.WaitOne(10 * BaseChangeFeedClientHelper.ChangeFeedSetupTime); await processor.StopAsync(); Assert.IsTrue(isStartOk, "Timed out waiting for docs to process"); Assert.AreEqual("doc0.doc1.doc2.doc3.doc4.", accumulator); }