private async Task <ResponseMessage> ReadNextInternalAsync(
CosmosDiagnosticsContext diagnostics,
CancellationToken cancellationToken = default)
{
cancellationToken.ThrowIfCancellationRequested();
Stream stream = null;
OperationType operation = OperationType.ReadFeed;
if (this.querySpec != null)
{
stream = this.clientContext.SerializerCore.ToStreamSqlQuerySpec(this.querySpec, this.resourceType);
operation = OperationType.Query;
}
if (this.feedTokenInternal == null)
{
TryCatch <FeedTokenInternal> tryCatchFeedTokeninternal = await this.TryInitializeFeedTokenAsync(cancellationToken);
if (!tryCatchFeedTokeninternal.Succeeded)
{
if (tryCatchFeedTokeninternal.Exception.InnerException is CosmosException cosmosException)
{
return(cosmosException.ToCosmosResponseMessage(new RequestMessage(method: null, requestUri: null, diagnosticsContext: diagnostics)));
}
return(CosmosExceptionFactory.CreateInternalServerErrorException(
message: tryCatchFeedTokeninternal.Exception.InnerException.Message,
innerException: tryCatchFeedTokeninternal.Exception.InnerException,
diagnosticsContext: diagnostics).ToCosmosResponseMessage(new RequestMessage(method: null, requestUri: null, diagnosticsContext: diagnostics)));
}
this.feedTokenInternal = tryCatchFeedTokeninternal.Result;
}
ResponseMessage response = await this.clientContext.ProcessResourceOperationStreamAsync(
resourceUri : this.resourceLink,
resourceType : this.resourceType,
operationType : operation,
requestOptions : this.requestOptions,
cosmosContainerCore : null,
partitionKey : this.requestOptions?.PartitionKey,
streamPayload : stream,
requestEnricher : request =>
{
QueryRequestOptions.FillContinuationToken(request, this.ContinuationToken);
if (this.querySpec != null)
{
request.Headers.Add(HttpConstants.HttpHeaders.ContentType, MediaTypes.QueryJson);
request.Headers.Add(HttpConstants.HttpHeaders.IsQuery, bool.TrueString);
}
this.feedTokenInternal?.EnrichRequest(request);
},
diagnosticsScope : diagnostics,
cancellationToken : cancellationToken);
// Retry in case of splits or other scenarios only on partitioned resources
if (this.containerCore != null &&
await this.feedTokenInternal.ShouldRetryAsync(this.containerCore, response, cancellationToken))
{
return(await this.ReadNextInternalAsync(diagnostics, cancellationToken));
}
if (response.IsSuccessStatusCode)
{
this.feedTokenInternal.UpdateContinuation(response.Headers.ContinuationToken);
this.ContinuationToken = this.feedTokenInternal.GetContinuation();
this.hasMoreResultsInternal = !this.feedTokenInternal.IsDone;
}
else
{
this.hasMoreResultsInternal = false;
}
return(response);
}
private async Task <ResponseMessage> ReadNextInternalAsync(
CosmosDiagnosticsContext diagnosticsScope,
CancellationToken cancellationToken = default(CancellationToken))
{
cancellationToken.ThrowIfCancellationRequested();
if (this.feedTokenInternal == null)
{
TryCatch <FeedTokenInternal> tryCatchFeedTokeninternal = await this.TryInitializeFeedTokenAsync(cancellationToken);
if (!tryCatchFeedTokeninternal.Succeeded)
{
if (tryCatchFeedTokeninternal.Exception.InnerException is CosmosException cosmosException)
{
return(cosmosException.ToCosmosResponseMessage(new RequestMessage(method: null, requestUri: null, diagnosticsContext: diagnosticsScope)));
}
return(CosmosExceptionFactory.CreateInternalServerErrorException(
message: tryCatchFeedTokeninternal.Exception.InnerException.Message,
innerException: tryCatchFeedTokeninternal.Exception.InnerException,
diagnosticsContext: diagnosticsScope).ToCosmosResponseMessage(new RequestMessage(method: null, requestUri: null, diagnosticsContext: diagnosticsScope)));
}
this.feedTokenInternal = tryCatchFeedTokeninternal.Result;
}
Uri resourceUri = this.container.LinkUri;
ResponseMessage responseMessage = await this.clientContext.ProcessResourceOperationStreamAsync(
resourceUri : resourceUri,
resourceType : Documents.ResourceType.Document,
operationType : Documents.OperationType.ReadFeed,
requestOptions : this.changeFeedOptions,
cosmosContainerCore : this.container,
requestEnricher : request =>
{
ChangeFeedRequestOptions.FillContinuationToken(request, this.feedTokenInternal.GetContinuation());
this.feedTokenInternal.EnrichRequest(request);
},
partitionKey : null,
streamPayload : null,
diagnosticsContext : diagnosticsScope,
cancellationToken : cancellationToken);
// Retry in case of splits or other scenarios
if (await this.feedTokenInternal.ShouldRetryAsync(this.container, responseMessage, cancellationToken))
{
if (responseMessage.IsSuccessStatusCode ||
responseMessage.StatusCode == HttpStatusCode.NotModified)
{
// Change Feed read uses Etag for continuation
this.feedTokenInternal.UpdateContinuation(responseMessage.Headers.ETag);
}
return(await this.ReadNextInternalAsync(diagnosticsScope, cancellationToken));
}
if (responseMessage.IsSuccessStatusCode ||
responseMessage.StatusCode == HttpStatusCode.NotModified)
{
// Change Feed read uses Etag for continuation
this.feedTokenInternal.UpdateContinuation(responseMessage.Headers.ETag);
}
this.hasMoreResults = responseMessage.IsSuccessStatusCode;
return(responseMessage);
}
private async Task <ResponseMessage> ReadNextInternalAsync(
CosmosDiagnosticsContext diagnostics,
CancellationToken cancellationToken = default)
{
cancellationToken.ThrowIfCancellationRequested();
Stream stream = null;
OperationType operation = OperationType.ReadFeed;
if (this.querySpec != null)
{
stream = this.clientContext.SerializerCore.ToStreamSqlQuerySpec(this.querySpec, this.resourceType);
operation = OperationType.Query;
}
ResponseMessage responseMessage = await this.clientContext.ProcessResourceOperationStreamAsync(
resourceUri : this.containerCore.LinkUri,
resourceType : this.resourceType,
operationType : operation,
requestOptions : this.queryRequestOptions,
cosmosContainerCore : this.containerCore,
partitionKey : this.queryRequestOptions?.PartitionKey,
streamPayload : stream,
requestEnricher : request =>
{
// FeedRangeContinuationRequestMessagePopulatorVisitor needs to run before FeedRangeRequestMessagePopulatorVisitor,
// since they both set EPK range headers and in the case of split we need to run on the child range and not the parent range.
FeedRangeContinuationRequestMessagePopulatorVisitor feedRangeContinuationVisitor = new FeedRangeContinuationRequestMessagePopulatorVisitor(
request,
QueryRequestOptions.FillContinuationToken);
this.FeedRangeContinuation.Accept(feedRangeContinuationVisitor);
FeedRangeRequestMessagePopulatorVisitor feedRangeVisitor = new FeedRangeRequestMessagePopulatorVisitor(request);
this.FeedRangeInternal.Accept(feedRangeVisitor);
if (this.querySpec != null)
{
request.Headers.Add(HttpConstants.HttpHeaders.ContentType, MediaTypes.QueryJson);
request.Headers.Add(HttpConstants.HttpHeaders.IsQuery, bool.TrueString);
}
},
diagnosticsContext : diagnostics,
cancellationToken : cancellationToken);
ShouldRetryResult shouldRetryOnSplit = await this.FeedRangeContinuation.HandleSplitAsync(this.containerCore, responseMessage, cancellationToken);
if (shouldRetryOnSplit.ShouldRetry)
{
return(await this.ReadNextInternalAsync(diagnostics, cancellationToken));
}
if (responseMessage.Content != null)
{
await CosmosElementSerializer.RewriteStreamAsTextAsync(responseMessage, this.queryRequestOptions);
}
if (responseMessage.IsSuccessStatusCode)
{
this.FeedRangeContinuation.ReplaceContinuation(responseMessage.Headers.ContinuationToken);
this.hasMoreResultsInternal = !this.FeedRangeContinuation.IsDone;
return(FeedRangeResponse.CreateSuccess(responseMessage, this.FeedRangeContinuation));
}
else
{
this.hasMoreResultsInternal = false;
return(FeedRangeResponse.CreateFailure(responseMessage));
}
}
// Extracted partition key might be invalid as CollectionCache might be stale.
// Stale collection cache is refreshed through PartitionKeyMismatchRetryPolicy
// and partition-key is extracted again.
private async Task <ResponseMessage> ExtractPartitionKeyAndProcessItemStreamAsync <T>(
PartitionKey?partitionKey,
string itemId,
T item,
OperationType operationType,
ItemRequestOptions requestOptions,
CosmosDiagnosticsContext diagnosticsContext,
CancellationToken cancellationToken)
{
if (diagnosticsContext == null)
{
throw new ArgumentNullException(nameof(diagnosticsContext));
}
Stream itemStream;
using (diagnosticsContext.CreateScope("ItemSerialize"))
{
itemStream = this.ClientContext.SerializerCore.ToStream <T>(item);
}
// User specified PK value, no need to extract it
if (partitionKey.HasValue)
{
PartitionKeyDefinition pKeyDefinition = await this.GetPartitionKeyDefinitionAsync();
if (partitionKey.HasValue && partitionKey.Value != PartitionKey.None && partitionKey.Value.InternalKey.Components.Count != pKeyDefinition.Paths.Count)
{
throw new ArgumentException(RMResources.MissingPartitionKeyValue);
}
return(await this.ProcessItemStreamAsync(
partitionKey,
itemId,
itemStream,
operationType,
requestOptions,
diagnosticsContext : diagnosticsContext,
cancellationToken : cancellationToken));
}
PartitionKeyMismatchRetryPolicy requestRetryPolicy = null;
while (true)
{
using (diagnosticsContext.CreateScope("ExtractPkValue"))
{
partitionKey = await this.GetPartitionKeyValueFromStreamAsync(itemStream, cancellationToken);
}
ResponseMessage responseMessage = await this.ProcessItemStreamAsync(
partitionKey,
itemId,
itemStream,
operationType,
requestOptions,
diagnosticsContext : diagnosticsContext,
cancellationToken : cancellationToken);
if (responseMessage.IsSuccessStatusCode)
{
return(responseMessage);
}
if (requestRetryPolicy == null)
{
requestRetryPolicy = new PartitionKeyMismatchRetryPolicy(await this.ClientContext.DocumentClient.GetCollectionCacheAsync(), null);
}
ShouldRetryResult retryResult = await requestRetryPolicy.ShouldRetryAsync(responseMessage, cancellationToken);
if (!retryResult.ShouldRetry)
{
return(responseMessage);
}
}
}
internal CosmosDiagnosticsCore(CosmosDiagnosticsContext diagnosticsContext)
{
this.Context = diagnosticsContext ?? throw new ArgumentNullException(nameof(diagnosticsContext));
}
internal abstract Task <Stream> DecryptItemAsync(
Stream input,
DatabaseCore database,
CosmosDiagnosticsContext diagnosticsContext,
CancellationToken cancellationToken);
/// <summary>
/// Create a <see cref="ResponseMessage"/>
/// </summary>
public ResponseMessage()
{
this.Headers = new Headers();
this.DiagnosticsContext = CosmosDiagnosticsContext.Create();
this.CosmosException = null;
}
private Task <HttpResponseMessage> SendHttpAsync(
Func <ValueTask <HttpRequestMessage> > createRequestMessageAsync,
HttpCompletionOption httpCompletionOption,
ResourceType resourceType,
CosmosDiagnosticsContext diagnosticsContext,
CancellationToken cancellationToken)
{
diagnosticsContext ??= new CosmosDiagnosticsContextCore();
HttpRequestMessage requestMessage = null;
Func <Task <HttpResponseMessage> > funcDelegate = async() =>
{
using (diagnosticsContext.CreateScope(nameof(CosmosHttpClientCore.SendHttpAsync)))
{
using (requestMessage = await createRequestMessageAsync())
{
DateTime sendTimeUtc = DateTime.UtcNow;
Guid localGuid = Guid.NewGuid(); // For correlating HttpRequest and HttpResponse Traces
Guid requestedActivityId = Trace.CorrelationManager.ActivityId;
this.eventSource.Request(
requestedActivityId,
localGuid,
requestMessage.RequestUri.ToString(),
resourceType.ToResourceTypeString(),
requestMessage.Headers);
HttpResponseMessage responseMessage = await this.httpClient.SendAsync(
requestMessage,
httpCompletionOption,
cancellationToken);
DateTime receivedTimeUtc = DateTime.UtcNow;
TimeSpan durationTimeSpan = receivedTimeUtc - sendTimeUtc;
Guid activityId = Guid.Empty;
if (responseMessage.Headers.TryGetValues(
HttpConstants.HttpHeaders.ActivityId,
out IEnumerable <string> headerValues) && headerValues.Any())
{
activityId = new Guid(headerValues.First());
}
this.eventSource.Response(
activityId,
localGuid,
(short)responseMessage.StatusCode,
durationTimeSpan.TotalMilliseconds,
responseMessage.Headers);
return(responseMessage);
}
}
};
HttpRequestMessage GetHttpRequestMessage() => requestMessage;
return(BackoffRetryUtility <HttpResponseMessage> .ExecuteAsync(
callbackMethod : funcDelegate,
retryPolicy : new TransientHttpClientRetryPolicy(
getHttpRequestMessage: GetHttpRequestMessage,
gatewayRequestTimeout: this.httpClient.Timeout,
diagnosticsContext: diagnosticsContext),
cancellationToken : cancellationToken));
}
public async Task <Stream> EncryptAsync(
Stream input,
EncryptionOptions encryptionOptions,
DatabaseCore database,
EncryptionKeyWrapProvider encryptionKeyWrapProvider,
CosmosDiagnosticsContext diagnosticsContext,
CancellationToken cancellationToken)
{
Debug.Assert(input != null);
Debug.Assert(encryptionOptions != null);
Debug.Assert(database != null);
Debug.Assert(diagnosticsContext != null);
if (encryptionOptions.PathsToEncrypt == null)
{
throw new ArgumentNullException(nameof(encryptionOptions.PathsToEncrypt));
}
if (encryptionOptions.PathsToEncrypt.Count == 0)
{
return(input);
}
foreach (string path in encryptionOptions.PathsToEncrypt)
{
if (string.IsNullOrEmpty(path) || path[0] != '/' || path.LastIndexOf('/') != 0)
{
throw new ArgumentException($"Invalid path {path ?? string.Empty}", nameof(encryptionOptions.PathsToEncrypt));
}
}
if (encryptionOptions.DataEncryptionKey == null)
{
throw new ArgumentException("Invalid encryption options", nameof(encryptionOptions.DataEncryptionKey));
}
if (encryptionKeyWrapProvider == null)
{
throw new ArgumentException(ClientResources.EncryptionKeyWrapProviderNotConfigured);
}
DataEncryptionKey dek = database.GetDataEncryptionKey(encryptionOptions.DataEncryptionKey.Id);
DataEncryptionKeyCore dekCore = (DataEncryptionKeyInlineCore)dek;
(DataEncryptionKeyProperties dekProperties, InMemoryRawDek inMemoryRawDek) = await dekCore.FetchUnwrappedAsync(
diagnosticsContext,
cancellationToken);
JObject itemJObj = EncryptionProcessor.baseSerializer.FromStream <JObject>(input);
JObject toEncryptJObj = new JObject();
foreach (string pathToEncrypt in encryptionOptions.PathsToEncrypt)
{
string propertyName = pathToEncrypt.Substring(1);
JToken propertyValueHolder = itemJObj.Property(propertyName).Value;
// Even null in the JSON is a JToken with Type Null, this null check is just a sanity check
if (propertyValueHolder != null)
{
toEncryptJObj.Add(propertyName, itemJObj.Property(propertyName).Value.Value <JToken>());
itemJObj.Remove(propertyName);
}
}
MemoryStream memoryStream = EncryptionProcessor.baseSerializer.ToStream <JObject>(toEncryptJObj) as MemoryStream;
Debug.Assert(memoryStream != null);
Debug.Assert(memoryStream.TryGetBuffer(out _));
byte[] plainText = memoryStream.GetBuffer();
EncryptionProperties encryptionProperties = new EncryptionProperties(
dataEncryptionKeyRid: dekProperties.ResourceId,
encryptionFormatVersion: 1,
encryptedData: inMemoryRawDek.AlgorithmUsingRawDek.EncryptData(plainText));
itemJObj.Add(Constants.Properties.EncryptedInfo, JObject.FromObject(encryptionProperties));
return(EncryptionProcessor.baseSerializer.ToStream(itemJObj));
}
public async Task <Stream> DecryptAsync(
Stream input,
DatabaseCore database,
EncryptionKeyWrapProvider encryptionKeyWrapProvider,
CosmosDiagnosticsContext diagnosticsContext,
CancellationToken cancellationToken)
{
Debug.Assert(input != null);
Debug.Assert(database != null);
Debug.Assert(input.CanSeek);
Debug.Assert(diagnosticsContext != null);
if (encryptionKeyWrapProvider == null)
{
return(input);
}
JObject itemJObj = EncryptionProcessor.baseSerializer.FromStream <JObject>(input);
JProperty encryptionPropertiesJProp = itemJObj.Property(Constants.Properties.EncryptedInfo);
JObject encryptionPropertiesJObj = null;
if (encryptionPropertiesJProp != null && encryptionPropertiesJProp.Value != null && encryptionPropertiesJProp.Value.Type == JTokenType.Object)
{
encryptionPropertiesJObj = (JObject)encryptionPropertiesJProp.Value;
}
if (encryptionPropertiesJObj == null)
{
input.Position = 0;
return(input);
}
EncryptionProperties encryptionProperties = encryptionPropertiesJObj.ToObject <EncryptionProperties>();
if (encryptionProperties.EncryptionFormatVersion != 1)
{
throw new CosmosException(HttpStatusCode.InternalServerError, $"Unknown encryption format version: {encryptionProperties.EncryptionFormatVersion}. Please upgrade your SDK to the latest version.");
}
DataEncryptionKeyCore tempDek = (DataEncryptionKeyInlineCore)database.GetDataEncryptionKey(id: "unknown");
(DataEncryptionKeyProperties _, InMemoryRawDek inMemoryRawDek) = await tempDek.FetchUnwrappedByRidAsync(
encryptionProperties.DataEncryptionKeyRid,
diagnosticsContext,
cancellationToken);
byte[] plainText = inMemoryRawDek.AlgorithmUsingRawDek.DecryptData(encryptionProperties.EncryptedData);
JObject plainTextJObj = null;
using (MemoryStream memoryStream = new MemoryStream(plainText))
using (StreamReader streamReader = new StreamReader(memoryStream))
using (JsonTextReader jsonTextReader = new JsonTextReader(streamReader))
{
plainTextJObj = JObject.Load(jsonTextReader);
}
foreach (JProperty property in plainTextJObj.Properties())
{
itemJObj.Add(property.Name, property.Value);
}
itemJObj.Remove(Constants.Properties.EncryptedInfo);
return(EncryptionProcessor.baseSerializer.ToStream(itemJObj));
}
public async Task <Stream> EncryptAsync(
Stream input,
EncryptionOptions encryptionOptions,
Encryptor encryptor,
CosmosDiagnosticsContext diagnosticsContext,
CancellationToken cancellationToken)
{
Debug.Assert(input != null);
Debug.Assert(encryptionOptions != null);
Debug.Assert(diagnosticsContext != null);
if (encryptor == null)
{
throw new ArgumentException(ClientResources.EncryptorNotConfigured);
}
if (string.IsNullOrEmpty(encryptionOptions.DataEncryptionKeyId))
{
throw new ArgumentNullException(nameof(encryptionOptions.DataEncryptionKeyId));
}
if (string.IsNullOrEmpty(encryptionOptions.EncryptionAlgorithm))
{
throw new ArgumentNullException(nameof(encryptionOptions.EncryptionAlgorithm));
}
if (encryptionOptions.PathsToEncrypt == null)
{
throw new ArgumentNullException(nameof(encryptionOptions.PathsToEncrypt));
}
if (encryptionOptions.PathsToEncrypt.Count == 0)
{
return(input);
}
foreach (string path in encryptionOptions.PathsToEncrypt)
{
if (string.IsNullOrEmpty(path) || path[0] != '/' || path.LastIndexOf('/') != 0)
{
throw new ArgumentException($"Invalid path {path ?? string.Empty}", nameof(encryptionOptions.PathsToEncrypt));
}
}
JObject itemJObj = EncryptionProcessor.baseSerializer.FromStream <JObject>(input);
JObject toEncryptJObj = new JObject();
foreach (string pathToEncrypt in encryptionOptions.PathsToEncrypt)
{
string propertyName = pathToEncrypt.Substring(1);
JToken propertyValueHolder = itemJObj.Property(propertyName).Value;
// Even null in the JSON is a JToken with Type Null, this null check is just a sanity check
if (propertyValueHolder != null)
{
toEncryptJObj.Add(propertyName, itemJObj.Property(propertyName).Value.Value <JToken>());
itemJObj.Remove(propertyName);
}
}
MemoryStream memoryStream = EncryptionProcessor.baseSerializer.ToStream <JObject>(toEncryptJObj) as MemoryStream;
Debug.Assert(memoryStream != null);
Debug.Assert(memoryStream.TryGetBuffer(out _));
byte[] plainText = memoryStream.GetBuffer();
byte[] cipherText = await encryptor.EncryptAsync(
plainText,
encryptionOptions.DataEncryptionKeyId,
encryptionOptions.EncryptionAlgorithm,
cancellationToken);
if (cipherText == null)
{
throw new InvalidOperationException($"{nameof(Encryptor)} returned null cipherText from {nameof(EncryptAsync)}.");
}
EncryptionProperties encryptionProperties = new EncryptionProperties(
encryptionFormatVersion: 2,
dataEncryptionKeyId: encryptionOptions.DataEncryptionKeyId,
encryptionAlgorithm: encryptionOptions.EncryptionAlgorithm,
encryptedData: cipherText);
itemJObj.Add(Constants.Properties.EncryptedInfo, JObject.FromObject(encryptionProperties));
return(EncryptionProcessor.baseSerializer.ToStream(itemJObj));
}