private async Task <ResponseMessage> ReadNextInternalAsync( CosmosDiagnosticsContext diagnostics, CancellationToken cancellationToken = default) { cancellationToken.ThrowIfCancellationRequested(); Stream stream = null; OperationType operation = OperationType.ReadFeed; if (this.querySpec != null) { stream = this.clientContext.SerializerCore.ToStreamSqlQuerySpec(this.querySpec, this.resourceType); operation = OperationType.Query; } if (this.feedTokenInternal == null) { TryCatch <FeedTokenInternal> tryCatchFeedTokeninternal = await this.TryInitializeFeedTokenAsync(cancellationToken); if (!tryCatchFeedTokeninternal.Succeeded) { if (tryCatchFeedTokeninternal.Exception.InnerException is CosmosException cosmosException) { return(cosmosException.ToCosmosResponseMessage(new RequestMessage(method: null, requestUri: null, diagnosticsContext: diagnostics))); } return(CosmosExceptionFactory.CreateInternalServerErrorException( message: tryCatchFeedTokeninternal.Exception.InnerException.Message, innerException: tryCatchFeedTokeninternal.Exception.InnerException, diagnosticsContext: diagnostics).ToCosmosResponseMessage(new RequestMessage(method: null, requestUri: null, diagnosticsContext: diagnostics))); } this.feedTokenInternal = tryCatchFeedTokeninternal.Result; } ResponseMessage response = await this.clientContext.ProcessResourceOperationStreamAsync( resourceUri : this.resourceLink, resourceType : this.resourceType, operationType : operation, requestOptions : this.requestOptions, cosmosContainerCore : null, partitionKey : this.requestOptions?.PartitionKey, streamPayload : stream, requestEnricher : request => { QueryRequestOptions.FillContinuationToken(request, this.ContinuationToken); if (this.querySpec != null) { request.Headers.Add(HttpConstants.HttpHeaders.ContentType, MediaTypes.QueryJson); request.Headers.Add(HttpConstants.HttpHeaders.IsQuery, bool.TrueString); } this.feedTokenInternal?.EnrichRequest(request); }, diagnosticsScope : diagnostics, cancellationToken : cancellationToken); // Retry in case of splits or other scenarios only on partitioned resources if (this.containerCore != null && await this.feedTokenInternal.ShouldRetryAsync(this.containerCore, response, cancellationToken)) { return(await this.ReadNextInternalAsync(diagnostics, cancellationToken)); } if (response.IsSuccessStatusCode) { this.feedTokenInternal.UpdateContinuation(response.Headers.ContinuationToken); this.ContinuationToken = this.feedTokenInternal.GetContinuation(); this.hasMoreResultsInternal = !this.feedTokenInternal.IsDone; } else { this.hasMoreResultsInternal = false; } return(response); }
private async Task <ResponseMessage> ReadNextInternalAsync( CosmosDiagnosticsContext diagnosticsScope, CancellationToken cancellationToken = default(CancellationToken)) { cancellationToken.ThrowIfCancellationRequested(); if (this.feedTokenInternal == null) { TryCatch <FeedTokenInternal> tryCatchFeedTokeninternal = await this.TryInitializeFeedTokenAsync(cancellationToken); if (!tryCatchFeedTokeninternal.Succeeded) { if (tryCatchFeedTokeninternal.Exception.InnerException is CosmosException cosmosException) { return(cosmosException.ToCosmosResponseMessage(new RequestMessage(method: null, requestUri: null, diagnosticsContext: diagnosticsScope))); } return(CosmosExceptionFactory.CreateInternalServerErrorException( message: tryCatchFeedTokeninternal.Exception.InnerException.Message, innerException: tryCatchFeedTokeninternal.Exception.InnerException, diagnosticsContext: diagnosticsScope).ToCosmosResponseMessage(new RequestMessage(method: null, requestUri: null, diagnosticsContext: diagnosticsScope))); } this.feedTokenInternal = tryCatchFeedTokeninternal.Result; } Uri resourceUri = this.container.LinkUri; ResponseMessage responseMessage = await this.clientContext.ProcessResourceOperationStreamAsync( resourceUri : resourceUri, resourceType : Documents.ResourceType.Document, operationType : Documents.OperationType.ReadFeed, requestOptions : this.changeFeedOptions, cosmosContainerCore : this.container, requestEnricher : request => { ChangeFeedRequestOptions.FillContinuationToken(request, this.feedTokenInternal.GetContinuation()); this.feedTokenInternal.EnrichRequest(request); }, partitionKey : null, streamPayload : null, diagnosticsContext : diagnosticsScope, cancellationToken : cancellationToken); // Retry in case of splits or other scenarios if (await this.feedTokenInternal.ShouldRetryAsync(this.container, responseMessage, cancellationToken)) { if (responseMessage.IsSuccessStatusCode || responseMessage.StatusCode == HttpStatusCode.NotModified) { // Change Feed read uses Etag for continuation this.feedTokenInternal.UpdateContinuation(responseMessage.Headers.ETag); } return(await this.ReadNextInternalAsync(diagnosticsScope, cancellationToken)); } if (responseMessage.IsSuccessStatusCode || responseMessage.StatusCode == HttpStatusCode.NotModified) { // Change Feed read uses Etag for continuation this.feedTokenInternal.UpdateContinuation(responseMessage.Headers.ETag); } this.hasMoreResults = responseMessage.IsSuccessStatusCode; return(responseMessage); }
private async Task <ResponseMessage> ReadNextInternalAsync( CosmosDiagnosticsContext diagnostics, CancellationToken cancellationToken = default) { cancellationToken.ThrowIfCancellationRequested(); Stream stream = null; OperationType operation = OperationType.ReadFeed; if (this.querySpec != null) { stream = this.clientContext.SerializerCore.ToStreamSqlQuerySpec(this.querySpec, this.resourceType); operation = OperationType.Query; } ResponseMessage responseMessage = await this.clientContext.ProcessResourceOperationStreamAsync( resourceUri : this.containerCore.LinkUri, resourceType : this.resourceType, operationType : operation, requestOptions : this.queryRequestOptions, cosmosContainerCore : this.containerCore, partitionKey : this.queryRequestOptions?.PartitionKey, streamPayload : stream, requestEnricher : request => { // FeedRangeContinuationRequestMessagePopulatorVisitor needs to run before FeedRangeRequestMessagePopulatorVisitor, // since they both set EPK range headers and in the case of split we need to run on the child range and not the parent range. FeedRangeContinuationRequestMessagePopulatorVisitor feedRangeContinuationVisitor = new FeedRangeContinuationRequestMessagePopulatorVisitor( request, QueryRequestOptions.FillContinuationToken); this.FeedRangeContinuation.Accept(feedRangeContinuationVisitor); FeedRangeRequestMessagePopulatorVisitor feedRangeVisitor = new FeedRangeRequestMessagePopulatorVisitor(request); this.FeedRangeInternal.Accept(feedRangeVisitor); if (this.querySpec != null) { request.Headers.Add(HttpConstants.HttpHeaders.ContentType, MediaTypes.QueryJson); request.Headers.Add(HttpConstants.HttpHeaders.IsQuery, bool.TrueString); } }, diagnosticsContext : diagnostics, cancellationToken : cancellationToken); ShouldRetryResult shouldRetryOnSplit = await this.FeedRangeContinuation.HandleSplitAsync(this.containerCore, responseMessage, cancellationToken); if (shouldRetryOnSplit.ShouldRetry) { return(await this.ReadNextInternalAsync(diagnostics, cancellationToken)); } if (responseMessage.Content != null) { await CosmosElementSerializer.RewriteStreamAsTextAsync(responseMessage, this.queryRequestOptions); } if (responseMessage.IsSuccessStatusCode) { this.FeedRangeContinuation.ReplaceContinuation(responseMessage.Headers.ContinuationToken); this.hasMoreResultsInternal = !this.FeedRangeContinuation.IsDone; return(FeedRangeResponse.CreateSuccess(responseMessage, this.FeedRangeContinuation)); } else { this.hasMoreResultsInternal = false; return(FeedRangeResponse.CreateFailure(responseMessage)); } }
// Extracted partition key might be invalid as CollectionCache might be stale. // Stale collection cache is refreshed through PartitionKeyMismatchRetryPolicy // and partition-key is extracted again. private async Task <ResponseMessage> ExtractPartitionKeyAndProcessItemStreamAsync <T>( PartitionKey?partitionKey, string itemId, T item, OperationType operationType, ItemRequestOptions requestOptions, CosmosDiagnosticsContext diagnosticsContext, CancellationToken cancellationToken) { if (diagnosticsContext == null) { throw new ArgumentNullException(nameof(diagnosticsContext)); } Stream itemStream; using (diagnosticsContext.CreateScope("ItemSerialize")) { itemStream = this.ClientContext.SerializerCore.ToStream <T>(item); } // User specified PK value, no need to extract it if (partitionKey.HasValue) { PartitionKeyDefinition pKeyDefinition = await this.GetPartitionKeyDefinitionAsync(); if (partitionKey.HasValue && partitionKey.Value != PartitionKey.None && partitionKey.Value.InternalKey.Components.Count != pKeyDefinition.Paths.Count) { throw new ArgumentException(RMResources.MissingPartitionKeyValue); } return(await this.ProcessItemStreamAsync( partitionKey, itemId, itemStream, operationType, requestOptions, diagnosticsContext : diagnosticsContext, cancellationToken : cancellationToken)); } PartitionKeyMismatchRetryPolicy requestRetryPolicy = null; while (true) { using (diagnosticsContext.CreateScope("ExtractPkValue")) { partitionKey = await this.GetPartitionKeyValueFromStreamAsync(itemStream, cancellationToken); } ResponseMessage responseMessage = await this.ProcessItemStreamAsync( partitionKey, itemId, itemStream, operationType, requestOptions, diagnosticsContext : diagnosticsContext, cancellationToken : cancellationToken); if (responseMessage.IsSuccessStatusCode) { return(responseMessage); } if (requestRetryPolicy == null) { requestRetryPolicy = new PartitionKeyMismatchRetryPolicy(await this.ClientContext.DocumentClient.GetCollectionCacheAsync(), null); } ShouldRetryResult retryResult = await requestRetryPolicy.ShouldRetryAsync(responseMessage, cancellationToken); if (!retryResult.ShouldRetry) { return(responseMessage); } } }
internal CosmosDiagnosticsCore(CosmosDiagnosticsContext diagnosticsContext) { this.Context = diagnosticsContext ?? throw new ArgumentNullException(nameof(diagnosticsContext)); }
internal abstract Task <Stream> DecryptItemAsync( Stream input, DatabaseCore database, CosmosDiagnosticsContext diagnosticsContext, CancellationToken cancellationToken);
/// <summary> /// Create a <see cref="ResponseMessage"/> /// </summary> public ResponseMessage() { this.Headers = new Headers(); this.DiagnosticsContext = CosmosDiagnosticsContext.Create(); this.CosmosException = null; }
private Task <HttpResponseMessage> SendHttpAsync( Func <ValueTask <HttpRequestMessage> > createRequestMessageAsync, HttpCompletionOption httpCompletionOption, ResourceType resourceType, CosmosDiagnosticsContext diagnosticsContext, CancellationToken cancellationToken) { diagnosticsContext ??= new CosmosDiagnosticsContextCore(); HttpRequestMessage requestMessage = null; Func <Task <HttpResponseMessage> > funcDelegate = async() => { using (diagnosticsContext.CreateScope(nameof(CosmosHttpClientCore.SendHttpAsync))) { using (requestMessage = await createRequestMessageAsync()) { DateTime sendTimeUtc = DateTime.UtcNow; Guid localGuid = Guid.NewGuid(); // For correlating HttpRequest and HttpResponse Traces Guid requestedActivityId = Trace.CorrelationManager.ActivityId; this.eventSource.Request( requestedActivityId, localGuid, requestMessage.RequestUri.ToString(), resourceType.ToResourceTypeString(), requestMessage.Headers); HttpResponseMessage responseMessage = await this.httpClient.SendAsync( requestMessage, httpCompletionOption, cancellationToken); DateTime receivedTimeUtc = DateTime.UtcNow; TimeSpan durationTimeSpan = receivedTimeUtc - sendTimeUtc; Guid activityId = Guid.Empty; if (responseMessage.Headers.TryGetValues( HttpConstants.HttpHeaders.ActivityId, out IEnumerable <string> headerValues) && headerValues.Any()) { activityId = new Guid(headerValues.First()); } this.eventSource.Response( activityId, localGuid, (short)responseMessage.StatusCode, durationTimeSpan.TotalMilliseconds, responseMessage.Headers); return(responseMessage); } } }; HttpRequestMessage GetHttpRequestMessage() => requestMessage; return(BackoffRetryUtility <HttpResponseMessage> .ExecuteAsync( callbackMethod : funcDelegate, retryPolicy : new TransientHttpClientRetryPolicy( getHttpRequestMessage: GetHttpRequestMessage, gatewayRequestTimeout: this.httpClient.Timeout, diagnosticsContext: diagnosticsContext), cancellationToken : cancellationToken)); }
public async Task <Stream> EncryptAsync( Stream input, EncryptionOptions encryptionOptions, DatabaseCore database, EncryptionKeyWrapProvider encryptionKeyWrapProvider, CosmosDiagnosticsContext diagnosticsContext, CancellationToken cancellationToken) { Debug.Assert(input != null); Debug.Assert(encryptionOptions != null); Debug.Assert(database != null); Debug.Assert(diagnosticsContext != null); if (encryptionOptions.PathsToEncrypt == null) { throw new ArgumentNullException(nameof(encryptionOptions.PathsToEncrypt)); } if (encryptionOptions.PathsToEncrypt.Count == 0) { return(input); } foreach (string path in encryptionOptions.PathsToEncrypt) { if (string.IsNullOrEmpty(path) || path[0] != '/' || path.LastIndexOf('/') != 0) { throw new ArgumentException($"Invalid path {path ?? string.Empty}", nameof(encryptionOptions.PathsToEncrypt)); } } if (encryptionOptions.DataEncryptionKey == null) { throw new ArgumentException("Invalid encryption options", nameof(encryptionOptions.DataEncryptionKey)); } if (encryptionKeyWrapProvider == null) { throw new ArgumentException(ClientResources.EncryptionKeyWrapProviderNotConfigured); } DataEncryptionKey dek = database.GetDataEncryptionKey(encryptionOptions.DataEncryptionKey.Id); DataEncryptionKeyCore dekCore = (DataEncryptionKeyInlineCore)dek; (DataEncryptionKeyProperties dekProperties, InMemoryRawDek inMemoryRawDek) = await dekCore.FetchUnwrappedAsync( diagnosticsContext, cancellationToken); JObject itemJObj = EncryptionProcessor.baseSerializer.FromStream <JObject>(input); JObject toEncryptJObj = new JObject(); foreach (string pathToEncrypt in encryptionOptions.PathsToEncrypt) { string propertyName = pathToEncrypt.Substring(1); JToken propertyValueHolder = itemJObj.Property(propertyName).Value; // Even null in the JSON is a JToken with Type Null, this null check is just a sanity check if (propertyValueHolder != null) { toEncryptJObj.Add(propertyName, itemJObj.Property(propertyName).Value.Value <JToken>()); itemJObj.Remove(propertyName); } } MemoryStream memoryStream = EncryptionProcessor.baseSerializer.ToStream <JObject>(toEncryptJObj) as MemoryStream; Debug.Assert(memoryStream != null); Debug.Assert(memoryStream.TryGetBuffer(out _)); byte[] plainText = memoryStream.GetBuffer(); EncryptionProperties encryptionProperties = new EncryptionProperties( dataEncryptionKeyRid: dekProperties.ResourceId, encryptionFormatVersion: 1, encryptedData: inMemoryRawDek.AlgorithmUsingRawDek.EncryptData(plainText)); itemJObj.Add(Constants.Properties.EncryptedInfo, JObject.FromObject(encryptionProperties)); return(EncryptionProcessor.baseSerializer.ToStream(itemJObj)); }
public async Task <Stream> DecryptAsync( Stream input, DatabaseCore database, EncryptionKeyWrapProvider encryptionKeyWrapProvider, CosmosDiagnosticsContext diagnosticsContext, CancellationToken cancellationToken) { Debug.Assert(input != null); Debug.Assert(database != null); Debug.Assert(input.CanSeek); Debug.Assert(diagnosticsContext != null); if (encryptionKeyWrapProvider == null) { return(input); } JObject itemJObj = EncryptionProcessor.baseSerializer.FromStream <JObject>(input); JProperty encryptionPropertiesJProp = itemJObj.Property(Constants.Properties.EncryptedInfo); JObject encryptionPropertiesJObj = null; if (encryptionPropertiesJProp != null && encryptionPropertiesJProp.Value != null && encryptionPropertiesJProp.Value.Type == JTokenType.Object) { encryptionPropertiesJObj = (JObject)encryptionPropertiesJProp.Value; } if (encryptionPropertiesJObj == null) { input.Position = 0; return(input); } EncryptionProperties encryptionProperties = encryptionPropertiesJObj.ToObject <EncryptionProperties>(); if (encryptionProperties.EncryptionFormatVersion != 1) { throw new CosmosException(HttpStatusCode.InternalServerError, $"Unknown encryption format version: {encryptionProperties.EncryptionFormatVersion}. Please upgrade your SDK to the latest version."); } DataEncryptionKeyCore tempDek = (DataEncryptionKeyInlineCore)database.GetDataEncryptionKey(id: "unknown"); (DataEncryptionKeyProperties _, InMemoryRawDek inMemoryRawDek) = await tempDek.FetchUnwrappedByRidAsync( encryptionProperties.DataEncryptionKeyRid, diagnosticsContext, cancellationToken); byte[] plainText = inMemoryRawDek.AlgorithmUsingRawDek.DecryptData(encryptionProperties.EncryptedData); JObject plainTextJObj = null; using (MemoryStream memoryStream = new MemoryStream(plainText)) using (StreamReader streamReader = new StreamReader(memoryStream)) using (JsonTextReader jsonTextReader = new JsonTextReader(streamReader)) { plainTextJObj = JObject.Load(jsonTextReader); } foreach (JProperty property in plainTextJObj.Properties()) { itemJObj.Add(property.Name, property.Value); } itemJObj.Remove(Constants.Properties.EncryptedInfo); return(EncryptionProcessor.baseSerializer.ToStream(itemJObj)); }
public async Task <Stream> EncryptAsync( Stream input, EncryptionOptions encryptionOptions, Encryptor encryptor, CosmosDiagnosticsContext diagnosticsContext, CancellationToken cancellationToken) { Debug.Assert(input != null); Debug.Assert(encryptionOptions != null); Debug.Assert(diagnosticsContext != null); if (encryptor == null) { throw new ArgumentException(ClientResources.EncryptorNotConfigured); } if (string.IsNullOrEmpty(encryptionOptions.DataEncryptionKeyId)) { throw new ArgumentNullException(nameof(encryptionOptions.DataEncryptionKeyId)); } if (string.IsNullOrEmpty(encryptionOptions.EncryptionAlgorithm)) { throw new ArgumentNullException(nameof(encryptionOptions.EncryptionAlgorithm)); } if (encryptionOptions.PathsToEncrypt == null) { throw new ArgumentNullException(nameof(encryptionOptions.PathsToEncrypt)); } if (encryptionOptions.PathsToEncrypt.Count == 0) { return(input); } foreach (string path in encryptionOptions.PathsToEncrypt) { if (string.IsNullOrEmpty(path) || path[0] != '/' || path.LastIndexOf('/') != 0) { throw new ArgumentException($"Invalid path {path ?? string.Empty}", nameof(encryptionOptions.PathsToEncrypt)); } } JObject itemJObj = EncryptionProcessor.baseSerializer.FromStream <JObject>(input); JObject toEncryptJObj = new JObject(); foreach (string pathToEncrypt in encryptionOptions.PathsToEncrypt) { string propertyName = pathToEncrypt.Substring(1); JToken propertyValueHolder = itemJObj.Property(propertyName).Value; // Even null in the JSON is a JToken with Type Null, this null check is just a sanity check if (propertyValueHolder != null) { toEncryptJObj.Add(propertyName, itemJObj.Property(propertyName).Value.Value <JToken>()); itemJObj.Remove(propertyName); } } MemoryStream memoryStream = EncryptionProcessor.baseSerializer.ToStream <JObject>(toEncryptJObj) as MemoryStream; Debug.Assert(memoryStream != null); Debug.Assert(memoryStream.TryGetBuffer(out _)); byte[] plainText = memoryStream.GetBuffer(); byte[] cipherText = await encryptor.EncryptAsync( plainText, encryptionOptions.DataEncryptionKeyId, encryptionOptions.EncryptionAlgorithm, cancellationToken); if (cipherText == null) { throw new InvalidOperationException($"{nameof(Encryptor)} returned null cipherText from {nameof(EncryptAsync)}."); } EncryptionProperties encryptionProperties = new EncryptionProperties( encryptionFormatVersion: 2, dataEncryptionKeyId: encryptionOptions.DataEncryptionKeyId, encryptionAlgorithm: encryptionOptions.EncryptionAlgorithm, encryptedData: cipherText); itemJObj.Add(Constants.Properties.EncryptedInfo, JObject.FromObject(encryptionProperties)); return(EncryptionProcessor.baseSerializer.ToStream(itemJObj)); }