private async ValueTask <Response> UpdateStatusAsync(bool async, CancellationToken cancellationToken)
{
if (!_hasCompleted)
{
using DiagnosticScope? scope = _diagnostics?.CreateScope($"{nameof(SparkSessionOperation)}.{nameof(UpdateStatus)}");
scope?.Start();
try
{
// Get the latest status
Response <SparkBatchJob> update = async
? await _client.GetSparkBatchJobAsync(_batchId, true, cancellationToken).ConfigureAwait(false)
: _client.GetSparkBatchJob(_batchId, true, cancellationToken);
// Check if the operation is no longer running
_hasCompleted = IsJobComplete(update.Value.Result ?? SparkBatchJobResultType.Uncertain, update.Value.State.Value, _completionType);
if (_hasCompleted)
{
_hasValue = true;
_value = update.Value;
}
// Update raw response
_rawResponse = update.GetRawResponse();
}
catch (Exception e)
{
scope?.Failed(e);
throw;
}
}
return(GetRawResponse());
}
/// <summary> Publishes a batch of CloudEvents to an Azure Event Grid topic. </summary>
/// <param name="events"> An array of events to be published to Event Grid. </param>
/// <param name="async">Whether to invoke the operation asynchronously.</param>
/// <param name="cancellationToken"> The cancellation token to use. </param>
private async Task <Response> SendCloudEventsInternal(IEnumerable <CloudEvent> events, bool async, CancellationToken cancellationToken = default)
{
using DiagnosticScope scope = _clientDiagnostics.CreateScope($"{nameof(EventGridPublisherClient)}.{nameof(SendEvents)}");
scope.Start();
try
{
// List of events cannot be null
Argument.AssertNotNull(events, nameof(events));
List <CloudEventInternal> eventsWithSerializedPayloads = new List <CloudEventInternal>();
foreach (CloudEvent cloudEvent in events)
{
// Individual events cannot be null
Argument.AssertNotNull(cloudEvent, nameof(cloudEvent));
CloudEventInternal newCloudEvent = new CloudEventInternal(
cloudEvent.Id,
cloudEvent.Source,
cloudEvent.Type,
"1.0")
{
Time = cloudEvent.Time,
DataBase64 = cloudEvent.DataBase64,
Datacontenttype = cloudEvent.DataContentType,
Dataschema = cloudEvent.DataSchema,
Subject = cloudEvent.Subject
};
foreach (KeyValuePair <string, object> kvp in cloudEvent.ExtensionAttributes)
{
newCloudEvent.Add(kvp.Key, new CustomModelSerializer(kvp.Value, _dataSerializer, cancellationToken));
}
// The 'Data' property is optional for CloudEvents
// Additionally, if the type of data is binary, 'Data' will not be populated (data will be stored in 'DataBase64' instead)
if (cloudEvent.Data != null)
{
MemoryStream stream = new MemoryStream();
_dataSerializer.Serialize(stream, cloudEvent.Data, cloudEvent.Data.GetType(), cancellationToken);
stream.Position = 0;
JsonDocument data = JsonDocument.Parse(stream);
newCloudEvent.Data = data.RootElement;
}
eventsWithSerializedPayloads.Add(newCloudEvent);
}
if (async)
{
// Publish asynchronously if called via an async path
return(await _serviceRestClient.PublishCloudEventEventsAsync(
_hostName,
eventsWithSerializedPayloads,
cancellationToken).ConfigureAwait(false));
}
else
{
return(_serviceRestClient.PublishCloudEventEvents(
_hostName,
eventsWithSerializedPayloads,
cancellationToken));
}
}
catch (Exception e)
{
scope.Failed(e);
throw;
}
}
/// <summary> Publishes a batch of EventGridEvents to an Azure Event Grid topic. </summary>
/// <param name="events"> An array of events to be published to Event Grid. </param>
/// <param name="async">Whether to invoke the operation asynchronously.</param>
/// <param name="cancellationToken"> The cancellation token to use. </param>
private async Task <Response> SendEventsInternal(IEnumerable <EventGridEvent> events, bool async, CancellationToken cancellationToken = default)
{
using DiagnosticScope scope = _clientDiagnostics.CreateScope($"{nameof(EventGridPublisherClient)}.{nameof(SendEvents)}");
scope.Start();
try
{
// List of events cannot be null
Argument.AssertNotNull(events, nameof(events));
List <EventGridEventInternal> eventsWithSerializedPayloads = new List <EventGridEventInternal>();
foreach (EventGridEvent egEvent in events)
{
// Individual events cannot be null
Argument.AssertNotNull(egEvent, nameof(egEvent));
JsonDocument data;
if (egEvent.Data is BinaryData binaryEventData)
{
try
{
data = JsonDocument.Parse(binaryEventData);
}
catch (JsonException)
{
data = SerializeObjectToJsonDocument(binaryEventData.ToString(), typeof(string), cancellationToken);
}
}
else
{
data = SerializeObjectToJsonDocument(egEvent.Data, egEvent.Data.GetType(), cancellationToken);
}
EventGridEventInternal newEGEvent = new EventGridEventInternal(
egEvent.Id,
egEvent.Subject,
data.RootElement,
egEvent.EventType,
egEvent.EventTime,
egEvent.DataVersion)
{
Topic = egEvent.Topic
};
eventsWithSerializedPayloads.Add(newEGEvent);
}
if (async)
{
// Publish asynchronously if called via an async path
return(await _serviceRestClient.PublishEventsAsync(
_hostName,
eventsWithSerializedPayloads,
cancellationToken).ConfigureAwait(false));
}
else
{
return(_serviceRestClient.PublishEvents(
_hostName,
eventsWithSerializedPayloads,
cancellationToken));
}
}
catch (Exception e)
{
scope.Failed(e);
throw;
}
}
/// <summary> Publishes a batch of CloudEvents to an Azure Event Grid topic. </summary>
/// <param name="events"> An array of events to be published to Event Grid. </param>
/// <param name="async">Whether to invoke the operation asynchronously</param>
/// <param name="cancellationToken"> The cancellation token to use. </param>
private async Task <Response> PublishCloudEventsInternal(IEnumerable <CloudEvent> events, bool async, CancellationToken cancellationToken = default)
{
using DiagnosticScope scope = _clientDiagnostics.CreateScope($"{nameof(EventGridPublisherClient)}.{nameof(PublishCloudEvents)}");
scope.Start();
try
{
// List of events cannot be null
Argument.AssertNotNull(events, nameof(events));
List <CloudEventInternal> eventsWithSerializedPayloads = new List <CloudEventInternal>();
foreach (CloudEvent cloudEvent in events)
{
// Individual events cannot be null
Argument.AssertNotNull(cloudEvent, nameof(cloudEvent));
CloudEventInternal newCloudEvent = new CloudEventInternal(
cloudEvent.Id,
cloudEvent.Source,
cloudEvent.Type,
cloudEvent.SpecVersion)
{
Time = cloudEvent.Time,
Datacontenttype = cloudEvent.DataContentType,
Dataschema = cloudEvent.DataSchema,
Subject = cloudEvent.Subject
};
foreach (KeyValuePair <string, object> kvp in cloudEvent.ExtensionAttributes)
{
newCloudEvent.Add(kvp.Key, new EventGridSerializer(kvp.Value, _serializer, cancellationToken));
}
// The 'Data' property is optional for CloudEvents
if (cloudEvent.Data != null)
{
if (cloudEvent.Data is IEnumerable <byte> enumerable)
{
newCloudEvent.DataBase64 = Convert.ToBase64String(enumerable.ToArray());
}
else if (cloudEvent.Data is ReadOnlyMemory <byte> memory)
{
newCloudEvent.DataBase64 = Convert.ToBase64String(memory.ToArray());
}
else
{
newCloudEvent.Data = new EventGridSerializer(
cloudEvent.Data,
_serializer,
cancellationToken);
}
}
eventsWithSerializedPayloads.Add(newCloudEvent);
}
if (async)
{
// Publish asynchronously if called via an async path
return(await _serviceRestClient.PublishCloudEventEventsAsync(
_hostName,
eventsWithSerializedPayloads,
cancellationToken).ConfigureAwait(false));
}
else
{
return(_serviceRestClient.PublishCloudEventEvents(
_hostName,
eventsWithSerializedPayloads,
cancellationToken));
}
}
catch (Exception e)
{
scope.Failed(e);
throw;
}
}
/// <summary>
/// Raise an <see cref="Azure.Core.SyncAsyncEventHandler{T}"/>
/// event by executing each of the handlers sequentially (to avoid
/// introducing accidental parallelism in customer code) and collecting
/// any exceptions.
/// </summary>
/// <typeparam name="T">Type of the event arguments.</typeparam>
/// <param name="eventHandler">The event's delegate.</param>
/// <param name="e">
/// An <see cref="SyncAsyncEventArgs"/> instance that contains the
/// event data.
/// </param>
/// <param name="declaringTypeName">
/// The name of the type declaring the event to construct a helpful
/// exception message and distributed tracing span.
/// </param>
/// <param name="eventName">
/// The name of the event to construct a helpful exception message and
/// distributed tracing span.
/// </param>
/// <param name="clientDiagnostics">
/// Client diagnostics to wrap all the handlers in a new distributed
/// tracing span.
/// </param>
/// <returns>
/// A task that represents running all of the event's handlers.
/// </returns>
/// <exception cref="AggregateException">
/// An exception was thrown during the execution of at least one of the
/// event's handlers.
/// </exception>
/// <exception cref="ArgumentNullException">
/// Thrown when <paramref name="e"/>, <paramref name="declaringTypeName"/>,
/// <paramref name="eventName"/>, or <paramref name="clientDiagnostics"/>
/// are null.
/// </exception>
/// <exception cref="ArgumentException">
/// Thrown when <paramref name="declaringTypeName"/> or
/// <paramref name="eventName"/> are empty.
/// </exception>
public static async Task RaiseAsync <T>(
this SyncAsyncEventHandler <T> eventHandler,
T e,
string declaringTypeName,
string eventName,
ClientDiagnostics clientDiagnostics)
where T : SyncAsyncEventArgs
{
Argument.AssertNotNull(e, nameof(e));
Argument.AssertNotNullOrEmpty(declaringTypeName, nameof(declaringTypeName));
Argument.AssertNotNullOrEmpty(eventName, nameof(eventName));
Argument.AssertNotNull(clientDiagnostics, nameof(clientDiagnostics));
// Get the invocation list, but return early if there's no work
if (eventHandler == null)
{
return;
}
Delegate[] handlers = eventHandler.GetInvocationList();
if (handlers == null || handlers.Length == 0)
{
return;
}
// Wrap handler invocation in a distributed tracing span so it's
// easy for customers to track and measure
string eventFullName = declaringTypeName + "." + eventName;
using DiagnosticScope scope = clientDiagnostics.CreateScope(eventFullName);
scope.Start();
try
{
// Collect any exceptions raised by handlers
List <Exception> failures = null;
// Raise the handlers sequentially so we don't introduce any
// unintentional parallelism in customer code
foreach (Delegate handler in handlers)
{
SyncAsyncEventHandler <T> azureHandler = (SyncAsyncEventHandler <T>)handler;
try
{
Task runHandlerTask = azureHandler(e);
// We can consider logging something when e.RunSynchronously
// is true, but runHandlerTask.IsComplete is false.
// (We'll not bother to check our tests because
// EnsureCompleted on the code path that raised the
// event will catch it for us.)
await runHandlerTask.ConfigureAwait(false);
}
catch (Exception ex)
{
failures ??= new List <Exception>();
failures.Add(ex);
}
}
// Wrap any exceptions in an AggregateException
if (failures?.Count > 0)
{
// Include the event name in the exception for easier debugging
throw new AggregateException(
"Unhandled exception(s) thrown when raising the " + eventFullName + " event.",
failures);
}
}
catch (Exception ex)
{
scope.Failed(ex);
throw;
}
}
/// <summary>
/// The main loop of a partition pump. It receives events from the Azure Event Hubs service
/// and delegates their processing to the inner partition processor.
/// </summary>
///
/// <param name="cancellationToken">A <see cref="CancellationToken"/> instance to signal the request to cancel the operation.</param>
///
/// <returns>A task to be resolved on when the operation has completed.</returns>
///
private async Task RunAsync(CancellationToken cancellationToken)
{
IEnumerable <EventData> receivedEvents;
Exception unrecoverableException = null;
// We'll break from the loop upon encountering a non-retriable exception. The event processor periodically
// checks its pumps' status, so it should be aware of when one of them stops working.
while (!cancellationToken.IsCancellationRequested)
{
try
{
receivedEvents = await InnerConsumer.ReceiveAsync(Options.MaximumMessageCount, Options.MaximumReceiveWaitTime, cancellationToken).ConfigureAwait(false);
using DiagnosticScope diagnosticScope = EventDataInstrumentation.ClientDiagnostics.CreateScope(DiagnosticProperty.EventProcessorProcessingActivityName);
diagnosticScope.AddAttribute("kind", "server");
if (diagnosticScope.IsEnabled)
{
foreach (var eventData in receivedEvents)
{
if (EventDataInstrumentation.TryExtractDiagnosticId(eventData, out string diagnosticId))
{
diagnosticScope.AddLink(diagnosticId);
}
}
}
diagnosticScope.Start();
try
{
await PartitionProcessor.ProcessEventsAsync(Context, receivedEvents, cancellationToken).ConfigureAwait(false);
}
catch (Exception partitionProcessorException)
{
diagnosticScope.Failed(partitionProcessorException);
unrecoverableException = partitionProcessorException;
CloseReason = PartitionProcessorCloseReason.PartitionProcessorException;
break;
}
}
catch (Exception eventHubException)
{
// Stop running only if it's not a retriable exception.
if (s_retryPolicy.CalculateRetryDelay(eventHubException, 1) == null)
{
unrecoverableException = eventHubException;
CloseReason = PartitionProcessorCloseReason.EventHubException;
break;
}
}
}
if (unrecoverableException != null)
{
// In case an exception is encountered while partition processor is processing the error, don't
// catch it and let the calling method (StopAsync) handle it.
await PartitionProcessor.ProcessErrorAsync(Context, unrecoverableException, cancellationToken).ConfigureAwait(false);
}
}
private Response <BlobContentInfo> UploadInSequence(
Stream content,
int blockSize,
BlobHttpHeaders blobHttpHeaders,
IDictionary <string, string> metadata,
BlobRequestConditions conditions,
IProgress <long> progressHandler,
AccessTier?accessTier,
CancellationToken cancellationToken)
{
// Wrap the staging and commit calls in an Upload span for
// distributed tracing
DiagnosticScope scope = _client.ClientDiagnostics.CreateScope(
_operationName ?? $"{nameof(Azure)}.{nameof(Storage)}.{nameof(Blobs)}.{nameof(BlobClient)}.{nameof(BlobClient.Upload)}");
try
{
scope.Start();
// Wrap progressHandler in a AggregatingProgressIncrementer to prevent
// progress from being reset with each stage blob operation.
if (progressHandler != null)
{
progressHandler = new AggregatingProgressIncrementer(progressHandler);
}
// The list tracking blocks IDs we're going to commit
List <string> blockIds = new List <string>();
// Partition the stream into individual blocks and stage them
IAsyncEnumerator <ChunkedStream> enumerator =
PartitionedUploadExtensions.GetBlocksAsync(content, blockSize, async: false, _arrayPool, cancellationToken)
.GetAsyncEnumerator(cancellationToken);
#pragma warning disable AZC0107
while (enumerator.MoveNextAsync().EnsureCompleted())
#pragma warning restore AZC0107
{
// Dispose the block after the loop iterates and return its
// memory to our ArrayPool
using ChunkedStream block = enumerator.Current;
// Stage the next block
string blockId = GenerateBlockId(block.AbsolutePosition);
_client.StageBlock(
blockId,
new MemoryStream(block.Bytes, 0, block.Length, writable: false),
conditions: conditions,
progressHandler: progressHandler,
cancellationToken: cancellationToken);
blockIds.Add(blockId);
}
// Commit the block list after everything has been staged to
// complete the upload
return(_client.CommitBlockList(
blockIds,
blobHttpHeaders,
metadata,
conditions,
accessTier,
cancellationToken));
}
catch (Exception ex)
{
scope.Failed(ex);
throw;
}
finally
{
scope.Dispose();
}
}
/// <summary>
/// Submit a <see cref="BlobBatch"/> of sub-operations.
/// </summary>
/// <param name="batch">
/// A <see cref="BlobBatch"/> of sub-operations.
/// </param>
/// <param name="throwOnAnyFailure">
/// A value indicating whether or not to throw exceptions for
/// sub-operation failures.
/// </param>
/// <param name="async">
/// Whether to invoke the operation asynchronously.
/// </param>
/// <param name="cancellationToken">
/// Optional <see cref="CancellationToken"/> to propagate notifications
/// that the operation should be cancelled.
/// </param>
/// <returns>
/// A <see cref="Response"/> on successfully submitting.
/// </returns>
/// <remarks>
/// A <see cref="RequestFailedException"/> will be thrown if
/// a failure to submit the batch occurs. Individual sub-operation
/// failures will only throw if <paramref name="throwOnAnyFailure"/> is
/// true and be wrapped in an <see cref="AggregateException"/>.
/// </remarks>
private async Task <Response> SubmitBatchInternal(
BlobBatch batch,
bool throwOnAnyFailure,
bool async,
CancellationToken cancellationToken)
{
DiagnosticScope scope = ClientDiagnostics.CreateScope($"{nameof(BlobBatchClient)}.{nameof(SubmitBatch)}");
try
{
scope.Start();
batch = batch ?? throw new ArgumentNullException(nameof(batch));
if (batch.Submitted)
{
throw BatchErrors.CannotResubmitBatch(nameof(batch));
}
else if (!batch.IsAssociatedClient(this))
{
throw BatchErrors.BatchClientDoesNotMatch(nameof(batch));
}
// Get the sub-operation messages to submit
IList <HttpMessage> messages = batch.GetMessagesToSubmit();
if (messages.Count == 0)
{
throw BatchErrors.CannotSubmitEmptyBatch(nameof(batch));
}
// TODO: Consider validating the upper limit of 256 messages
// Merge the sub-operations into a single multipart/mixed Stream
(Stream content, string contentType) =
await MergeOperationRequests(
messages,
async,
cancellationToken)
.ConfigureAwait(false);
if (IsContainerScoped)
{
ResponseWithHeaders <Stream, ContainerSubmitBatchHeaders> response;
if (async)
{
response = await _containerRestClient.SubmitBatchAsync(
containerName : ContainerName,
contentLength : content.Length,
multipartContentType : contentType,
body : content,
cancellationToken : cancellationToken)
.ConfigureAwait(false);
}
else
{
response = _containerRestClient.SubmitBatch(
containerName: ContainerName,
contentLength: content.Length,
multipartContentType: contentType,
body: content,
cancellationToken: cancellationToken);
}
await UpdateOperationResponses(
messages,
response.GetRawResponse(),
response.Value,
response.Headers.ContentType,
throwOnAnyFailure,
async,
cancellationToken)
.ConfigureAwait(false);
return(response.GetRawResponse());
}
else
{
ResponseWithHeaders <Stream, ServiceSubmitBatchHeaders> response;
if (async)
{
response = await _serviceRestClient.SubmitBatchAsync(
contentLength : content.Length,
multipartContentType : contentType,
body : content,
cancellationToken : cancellationToken)
.ConfigureAwait(false);
}
else
{
response = _serviceRestClient.SubmitBatch(
contentLength: content.Length,
multipartContentType: contentType,
body: content,
cancellationToken: cancellationToken);
}
await UpdateOperationResponses(
messages,
response.GetRawResponse(),
response.Value,
response.Headers.ContentType,
throwOnAnyFailure,
async,
cancellationToken)
.ConfigureAwait(false);
return(response.GetRawResponse());
}
}
catch (Exception ex)
{
scope.Failed(ex);
throw;
}
finally
{
scope.Dispose();
}
}
/// <summary>
/// The <see cref="BreakInternal"/> operation breaks the files's
/// previously-acquired lease (if it exists).
///
/// Once a lease is broken, it cannot be renewed. Any authorized
/// request can break the lease; the request is not required to
/// specify a matching lease ID.
///
/// A lease that has been broken can also be released. A client can
/// immediately acquire a file lease that has been
/// released.
///
/// </summary>
/// <param name="async">
/// Whether to invoke the operation asynchronously.
/// </param>
/// <param name="cancellationToken">
/// Optional <see cref="CancellationToken"/> to propagate
/// notifications that the operation should be cancelled.
/// </param>
/// <returns>
/// A <see cref="Response{FileLease}"/> describing the broken lease.
/// </returns>
/// <remarks>
/// A <see cref="RequestFailedException"/> will be thrown if
/// a failure occurs.
/// </remarks>
private async Task <Response <ShareFileLease> > BreakInternal(
bool async,
CancellationToken cancellationToken)
{
EnsureClient();
using (ClientConfiguration.Pipeline.BeginLoggingScope(nameof(ShareLeaseClient)))
{
ClientConfiguration.Pipeline.LogMethodEnter(
nameof(ShareLeaseClient),
message:
$"{nameof(Uri)}: {Uri}\n" +
$"{nameof(LeaseId)}: {LeaseId}");
DiagnosticScope scope = ClientConfiguration.ClientDiagnostics.CreateScope($"{nameof(ShareLeaseClient)}.{nameof(Break)}");
try
{
scope.Start();
if (FileClient != null)
{
ResponseWithHeaders <FileBreakLeaseHeaders> response;
if (async)
{
response = await FileClient.FileRestClient.BreakLeaseAsync(
leaseAccessConditions : null,
cancellationToken : cancellationToken)
.ConfigureAwait(false);
}
else
{
response = FileClient.FileRestClient.BreakLease(
leaseAccessConditions: null,
cancellationToken: cancellationToken);
}
return(Response.FromValue(
response.ToShareFileLease(),
response.GetRawResponse()));
}
else
{
ResponseWithHeaders <ShareBreakLeaseHeaders> response;
if (async)
{
response = await ShareClient.ShareRestClient.BreakLeaseAsync(
breakPeriod : null,
leaseAccessConditions : null,
cancellationToken : cancellationToken)
.ConfigureAwait(false);
}
else
{
response = ShareClient.ShareRestClient.BreakLease(
breakPeriod: null,
leaseAccessConditions: null,
cancellationToken: cancellationToken);
}
return(Response.FromValue(
response.ToShareFileLease(),
response.GetRawResponse()));
}
}
catch (Exception ex)
{
ClientConfiguration.Pipeline.LogException(ex);
scope.Failed(ex);
throw;
}
finally
{
ClientConfiguration.Pipeline.LogMethodExit(nameof(ShareLeaseClient));
scope.Dispose();
}
}
}
/// <summary>
/// The <see cref="AcquireInternal"/> operation acquires a lease on
/// the file.
///
/// If the file does not have an active lease, the File service
/// creates a lease on the file and returns it. If the
/// file has an active lease, you can only request a new lease
/// using the active lease ID as <see cref="LeaseId"/>.
///
/// </summary>
/// <param name="duration">
/// Specifies the duration of the lease, in seconds, or specify
/// <see cref="InfiniteLeaseDuration"/> for a lease that never expires.
/// A non-infinite lease can be between 15 and 60 seconds.
/// A lease duration cannot be changed using <see cref="RenewAsync"/>
/// or <see cref="ChangeAsync"/>.
/// </param>
/// <param name="async">
/// Whether to invoke the operation asynchronously.
/// </param>
/// <param name="cancellationToken">
/// Optional <see cref="CancellationToken"/> to propagate
/// notifications that the operation should be cancelled.
/// </param>
/// <returns>
/// A <see cref="Response{Lease}"/> describing the lease.
/// </returns>
/// <remarks>
/// A <see cref="RequestFailedException"/> will be thrown if
/// a failure occurs.
/// </remarks>
private async Task <Response <ShareFileLease> > AcquireInternal(
TimeSpan?duration,
bool async,
CancellationToken cancellationToken)
{
EnsureClient();
using (ClientConfiguration.Pipeline.BeginLoggingScope(nameof(ShareLeaseClient)))
{
ClientConfiguration.Pipeline.LogMethodEnter(
nameof(ShareLeaseClient),
message:
$"{nameof(Uri)}: {Uri}\n" +
$"{nameof(LeaseId)}: {LeaseId}\n");
DiagnosticScope scope = ClientConfiguration.ClientDiagnostics.CreateScope($"{nameof(ShareLeaseClient)}.{nameof(Acquire)}");
try
{
scope.Start();
if (FileClient != null)
{
ResponseWithHeaders <FileAcquireLeaseHeaders> response;
if (async)
{
response = await FileClient.FileRestClient.AcquireLeaseAsync(
duration : (int)Constants.File.Lease.InfiniteLeaseDuration,
proposedLeaseId : LeaseId,
cancellationToken : cancellationToken)
.ConfigureAwait(false);
}
else
{
response = FileClient.FileRestClient.AcquireLease(
duration: (int)Constants.File.Lease.InfiniteLeaseDuration,
proposedLeaseId: LeaseId,
cancellationToken: cancellationToken);
}
return(Response.FromValue(
response.ToShareFileLease(),
response.GetRawResponse()));
}
else
{
// Int64 is an overflow safe cast relative to TimeSpan.MaxValue
long serviceDuration;
if (duration.HasValue && duration.Value >= TimeSpan.Zero)
{
serviceDuration = Convert.ToInt64(duration.Value.TotalSeconds);
}
else
{
serviceDuration = Constants.File.Lease.InfiniteLeaseDuration;
}
ResponseWithHeaders <ShareAcquireLeaseHeaders> response;
if (async)
{
response = await ShareClient.ShareRestClient.AcquireLeaseAsync(
duration : (int)serviceDuration,
proposedLeaseId : LeaseId,
cancellationToken : cancellationToken)
.ConfigureAwait(false);
}
else
{
response = ShareClient.ShareRestClient.AcquireLease(
duration: (int)serviceDuration,
proposedLeaseId: LeaseId,
cancellationToken: cancellationToken);
}
return(Response.FromValue(
response.ToShareFileLease(),
response.GetRawResponse()));
}
}
catch (Exception ex)
{
ClientConfiguration.Pipeline.LogException(ex);
scope.Failed(ex);
throw;
}
finally
{
ClientConfiguration.Pipeline.LogMethodExit(nameof(ShareLeaseClient));
scope.Dispose();
}
}
}
private static async Task UpdateClientsideKeyEncryptionKeyInternal(
BlobClient client,
UpdateClientSideKeyEncryptionKeyOptions options,
bool async,
CancellationToken cancellationToken)
{
// argument validation
Argument.AssertNotNull(client, nameof(client));
ClientSideEncryptionOptions operationEncryptionOptions = options?.EncryptionOptionsOverride
?? client.ClientSideEncryption
?? throw new ArgumentException($"{nameof(ClientSideEncryptionOptions)} are not configured on this client and none were provided for the operation.");
Argument.AssertNotNull(operationEncryptionOptions.KeyEncryptionKey, nameof(ClientSideEncryptionOptions.KeyEncryptionKey));
Argument.AssertNotNull(operationEncryptionOptions.KeyResolver, nameof(ClientSideEncryptionOptions.KeyResolver));
Argument.AssertNotNull(operationEncryptionOptions.KeyWrapAlgorithm, nameof(ClientSideEncryptionOptions.KeyWrapAlgorithm));
using (client.ClientConfiguration.Pipeline.BeginLoggingScope(nameof(BlobClient)))
{
client.ClientConfiguration.Pipeline.LogMethodEnter(
nameof(BlobBaseClient),
message:
$"{nameof(Uri)}: {client.Uri}\n" +
$"{nameof(options.Conditions)}: {options?.Conditions}");
DiagnosticScope scope = client.ClientConfiguration.ClientDiagnostics.CreateScope($"{nameof(BlobClient)}.{nameof(UpdateClientSideKeyEncryptionKey)}");
try
{
// hold onto etag, metadata, encryptiondata
BlobProperties getPropertiesResponse = await client.GetPropertiesInternal(options?.Conditions, async, cancellationToken).ConfigureAwait(false);
ETag etag = getPropertiesResponse.ETag;
IDictionary <string, string> metadata = getPropertiesResponse.Metadata;
EncryptionData encryptionData = BlobClientSideDecryptor.GetAndValidateEncryptionDataOrDefault(metadata)
?? throw new InvalidOperationException("Resource has no client-side encryption key to rotate.");
// rotate keywrapping
byte[] newWrappedKey = await WrapKeyInternal(
await UnwrapKeyInternal(
encryptionData,
operationEncryptionOptions.KeyResolver,
async,
cancellationToken).ConfigureAwait(false),
operationEncryptionOptions.KeyWrapAlgorithm,
operationEncryptionOptions.KeyEncryptionKey,
async,
cancellationToken).ConfigureAwait(false);
// set new wrapped key info and reinsert into metadata
encryptionData.WrappedContentKey = new KeyEnvelope
{
EncryptedKey = newWrappedKey,
Algorithm = operationEncryptionOptions.KeyWrapAlgorithm,
KeyId = operationEncryptionOptions.KeyEncryptionKey.KeyId
};
metadata[Constants.ClientSideEncryption.EncryptionDataKey] = EncryptionDataSerializer.Serialize(encryptionData);
// update blob ONLY IF ETAG MATCHES (do not take chances encryption info is now out of sync)
BlobRequestConditions modifiedRequestConditions = BlobRequestConditions.CloneOrDefault(options?.Conditions) ?? new BlobRequestConditions();
modifiedRequestConditions.IfMatch = etag;
await client.SetMetadataInternal(metadata, modifiedRequestConditions, async, cancellationToken).ConfigureAwait(false);
}
catch (Exception ex)
{
client.ClientConfiguration.Pipeline.LogException(ex);
scope.Failed(ex);
throw;
}
finally
{
client.ClientConfiguration.Pipeline.LogMethodExit(nameof(BlobBaseClient));
scope.Dispose();
}
}
}
/// <summary>
/// Returns a single segment of containers starting from the specified marker.
///
/// For more information, see
/// <see href="https://docs.microsoft.com/rest/api/storageservices/list-queues1">
/// List Queues</see>.
/// </summary>
/// <param name="marker">
/// Marker from the previous request.
/// </param>
/// <param name="traits">
/// Optional trait options for shaping the queues.
/// </param>
/// <param name="prefix">
/// Optional string that filters the results to return only queues
/// whose name begins with the specified <paramref name="prefix"/>.
/// </param>
/// <param name="pageSizeHint">
/// Optional hint to specify the desired size of the page returned.
/// </param>
/// <param name="async">
/// Whether to invoke the operation asynchronously.
/// </param>
/// <param name="cancellationToken">
/// <see cref="CancellationToken"/>
/// </param>
/// <returns>
/// A single segment of containers starting from the specified marker, including the next marker if appropriate.
/// </returns>
/// <remarks>
/// Use an empty marker to start enumeration from the beginning. Queue names are returned in lexicographic order.
/// After getting a segment, process it, and then call ListQueuesSegmentAsync again (passing in the next marker) to get the next segment.
/// </remarks>
internal async Task <Response <ListQueuesSegmentResponse> > GetQueuesInternal(
string marker,
QueueTraits traits,
string prefix,
int?pageSizeHint,
bool async,
CancellationToken cancellationToken)
{
using (ClientConfiguration.Pipeline.BeginLoggingScope(nameof(QueueServiceClient)))
{
ClientConfiguration.Pipeline.LogMethodEnter(
nameof(QueueServiceClient),
message:
$"{nameof(Uri)}: {Uri}\n" +
$"{nameof(marker)}: {marker}\n" +
$"{nameof(traits)}: {traits}\n" +
$"{nameof(prefix)}: {prefix}");
DiagnosticScope scope = ClientConfiguration.ClientDiagnostics.CreateScope($"{nameof(QueueServiceClient)}.{nameof(GetQueues)}");
try
{
ResponseWithHeaders <ListQueuesSegmentResponse, ServiceListQueuesSegmentHeaders> response;
scope.Start();
IEnumerable <string> includeTypes = traits.AsIncludeTypes();
if (async)
{
response = await _serviceRestClient.ListQueuesSegmentAsync(
prefix : prefix,
marker : marker,
maxresults : pageSizeHint,
include : includeTypes.Any()?includeTypes : null,
cancellationToken : cancellationToken)
.ConfigureAwait(false);
}
else
{
response = _serviceRestClient.ListQueuesSegment(
prefix: prefix,
marker: marker,
maxresults: pageSizeHint,
include: includeTypes.Any() ? includeTypes : null,
cancellationToken: cancellationToken);
}
if ((traits & QueueTraits.Metadata) != QueueTraits.Metadata)
{
IEnumerable <QueueItem> queueItems = response.Value.QueueItems;
foreach (QueueItem queueItem in queueItems)
{
queueItem.Metadata = null;
}
}
return(response);
}
catch (Exception ex)
{
ClientConfiguration.Pipeline.LogException(ex);
scope.Failed(ex);
throw;
}
finally
{
ClientConfiguration.Pipeline.LogMethodExit(nameof(QueueServiceClient));
scope.Dispose();
}
}
}
/// <summary>
/// The main loop of a partition pump. It receives events from the Azure Event Hubs service
/// and delegates their processing to the event processor processing handlers.
/// </summary>
///
/// <param name="cancellationToken">A <see cref="CancellationToken"/> instance to signal the request to cancel the operation.</param>
///
/// <returns>A task to be resolved on when the operation has completed.</returns>
///
private async Task RunAsync(CancellationToken cancellationToken)
{
List <EventData> receivedEvents;
Exception unrecoverableException = null;
// We'll break from the loop upon encountering a non-retriable exception. The event processor periodically
// checks its pumps' status, so it should be aware of when one of them stops working.
while (!cancellationToken.IsCancellationRequested)
{
await using (var partitionReceiver = InnerConsumer.CreatePartitionReceiver(Context.PartitionId, StartingPosition))
{
try
{
receivedEvents = (await partitionReceiver.ReceiveAsync(MaximumMessageCount, Options.MaximumReceiveWaitTime, cancellationToken).ConfigureAwait(false)).ToList();
using DiagnosticScope diagnosticScope = EventDataInstrumentation.ClientDiagnostics.CreateScope(DiagnosticProperty.EventProcessorProcessingActivityName);
diagnosticScope.AddAttribute("kind", "server");
if (diagnosticScope.IsEnabled)
{
foreach (var eventData in receivedEvents)
{
if (EventDataInstrumentation.TryExtractDiagnosticId(eventData, out string diagnosticId))
{
diagnosticScope.AddLink(diagnosticId);
}
}
}
// Small workaround to make sure we call ProcessEvent with EventData = null when no events have been received.
// The code is expected to get simpler when we start using the async enumerator internally to receive events.
if (receivedEvents.Count == 0)
{
receivedEvents.Add(null);
}
diagnosticScope.Start();
foreach (var eventData in receivedEvents)
{
try
{
var processorEvent = new EventProcessorEvent(Context, eventData, UpdateCheckpointAsync);
await ProcessEventAsync(processorEvent).ConfigureAwait(false);
}
catch (Exception eventProcessingException)
{
diagnosticScope.Failed(eventProcessingException);
unrecoverableException = eventProcessingException;
break;
}
}
}
catch (Exception eventHubException)
{
// Stop running only if it's not a retriable exception.
if (RetryPolicy.CalculateRetryDelay(eventHubException, 1) == null)
{
throw eventHubException;
}
}
if (unrecoverableException != null)
{
throw unrecoverableException;
}
}
}
}
/// <summary>
/// Submit a <see cref="BlobBatch"/> of sub-operations.
/// </summary>
/// <param name="batch">
/// A <see cref="BlobBatch"/> of sub-operations.
/// </param>
/// <param name="throwOnAnyFailure">
/// A value indicating whether or not to throw exceptions for
/// sub-operation failures.
/// </param>
/// <param name="async">
/// Whether to invoke the operation asynchronously.
/// </param>
/// <param name="cancellationToken">
/// Optional <see cref="CancellationToken"/> to propagate notifications
/// that the operation should be cancelled.
/// </param>
/// <returns>
/// A <see cref="Response"/> on successfully submitting.
/// </returns>
/// <remarks>
/// A <see cref="RequestFailedException"/> will be thrown if
/// a failure to submit the batch occurs. Individual sub-operation
/// failures will only throw if <paramref name="throwOnAnyFailure"/> is
/// true and be wrapped in an <see cref="AggregateException"/>.
/// </remarks>
private async Task <Response> SubmitBatchInternal(
BlobBatch batch,
bool throwOnAnyFailure,
bool async,
CancellationToken cancellationToken)
{
DiagnosticScope scope = ClientDiagnostics.CreateScope($"{nameof(BlobBatchClient)}.{nameof(SubmitBatch)}");
try
{
scope.Start();
batch = batch ?? throw new ArgumentNullException(nameof(batch));
if (batch.Submitted)
{
throw BatchErrors.CannotResubmitBatch(nameof(batch));
}
else if (!batch.IsAssociatedClient(this))
{
throw BatchErrors.BatchClientDoesNotMatch(nameof(batch));
}
// Get the sub-operation messages to submit
IList <HttpMessage> messages = batch.GetMessagesToSubmit();
if (messages.Count == 0)
{
throw BatchErrors.CannotSubmitEmptyBatch(nameof(batch));
}
// TODO: Consider validating the upper limit of 256 messages
// Merge the sub-operations into a single multipart/mixed Stream
(Stream content, string contentType) =
await MergeOperationRequests(
messages,
async,
cancellationToken)
.ConfigureAwait(false);
// Send the batch request
Response <BlobBatchResult> batchResult;
if (_isContainerScoped)
{
batchResult = await BatchRestClient.Container.SubmitBatchAsync(
clientDiagnostics : ClientDiagnostics,
pipeline : Pipeline,
resourceUri : Uri,
body : content,
contentLength : content.Length,
multipartContentType : contentType,
version : Version.ToVersionString(),
async : async,
operationName : $"{nameof(BlobBatchClient)}.{nameof(SubmitBatch)}",
cancellationToken : cancellationToken)
.ConfigureAwait(false);
}
else
{
batchResult = await BatchRestClient.Service.SubmitBatchAsync(
clientDiagnostics : ClientDiagnostics,
pipeline : Pipeline,
resourceUri : Uri,
body : content,
contentLength : content.Length,
multipartContentType : contentType,
version : Version.ToVersionString(),
async : async,
operationName : $"{nameof(BlobBatchClient)}.{nameof(SubmitBatch)}",
cancellationToken : cancellationToken)
.ConfigureAwait(false);
}
// Split the responses apart and update the sub-operation responses
Response raw = batchResult.GetRawResponse();
await UpdateOperationResponses(
messages,
raw,
batchResult.Value.Content,
batchResult.Value.ContentType,
throwOnAnyFailure,
async,
cancellationToken)
.ConfigureAwait(false);
// Return the batch result
return(raw);
}
catch (Exception ex)
{
scope.Failed(ex);
throw;
}
finally
{
scope.Dispose();
}
}
public Response DownloadTo(
Stream destination,
BlobRequestConditions conditions,
CancellationToken cancellationToken)
{
// Wrap the download range calls in a Download span for distributed
// tracing
DiagnosticScope scope = _client.ClientDiagnostics.CreateScope($"{nameof(BlobBaseClient)}.{nameof(BlobBaseClient.DownloadTo)}");
try
{
scope.Start();
// Just start downloading using an initial range. If it's a
// small blob, we'll get the whole thing in one shot. If it's
// a large blob, we'll get its full size in Content-Range and
// can keep downloading it in segments.
var initialRange = new HttpRange(0, _initialRangeSize);
Response <BlobDownloadInfo> initialResponse = _client.Download(
initialRange,
conditions,
rangeGetContentHash: false,
cancellationToken);
// If the initial request returned no content (i.e., a 304),
// we'll pass that back to the user immediately
if (initialResponse.IsUnavailable())
{
return(initialResponse.GetRawResponse());
}
// Copy the first segment to the destination stream
CopyTo(initialResponse, destination, cancellationToken);
// If the first segment was the entire blob, we're finished now
long initialLength = initialResponse.Value.ContentLength;
long totalLength = ParseRangeTotalLength(initialResponse.Value.Details.ContentRange);
if (initialLength == totalLength)
{
return(initialResponse.GetRawResponse());
}
// Capture the etag from the first segment and construct
// conditions to ensure the blob doesn't change while we're
// downloading the remaining segments
ETag etag = initialResponse.Value.Details.ETag;
BlobRequestConditions conditionsWithEtag = CreateConditionsWithEtag(conditions, etag);
// Download each of the remaining ranges in the blob
foreach (HttpRange httpRange in GetRanges(initialLength, totalLength))
{
// Don't need to worry about 304s here because the ETag
// condition will turn into a 412 and throw a proper
// RequestFailedException
Response <BlobDownloadInfo> result = _client.Download(
httpRange,
conditionsWithEtag,
rangeGetContentHash: false,
cancellationToken);
CopyTo(result.Value, destination, cancellationToken);
}
return(initialResponse.GetRawResponse());
}
catch (Exception ex)
{
scope.Failed(ex);
throw;
}
finally
{
scope.Dispose();
}
}
public async Task <Response> DownloadToAsync(
Stream destination,
BlobRequestConditions conditions,
CancellationToken cancellationToken)
{
// Wrap the download range calls in a Download span for distributed
// tracing
DiagnosticScope scope = _client.ClientDiagnostics.CreateScope($"{nameof(BlobBaseClient)}.{nameof(BlobBaseClient.DownloadTo)}");
try
{
scope.Start();
// Just start downloading using an initial range. If it's a
// small blob, we'll get the whole thing in one shot. If it's
// a large blob, we'll get its full size in Content-Range and
// can keep downloading it in segments.
var initialRange = new HttpRange(0, _initialRangeSize);
Task <Response <BlobDownloadInfo> > initialResponseTask =
_client.DownloadAsync(
initialRange,
conditions,
rangeGetContentHash: false,
cancellationToken);
Response <BlobDownloadInfo> initialResponse =
await initialResponseTask.ConfigureAwait(false);
// If the initial request returned no content (i.e., a 304),
// we'll pass that back to the user immediately
if (initialResponse.IsUnavailable())
{
return(initialResponse.GetRawResponse());
}
// If the first segment was the entire blob, we'll copy that to
// the output stream and finish now
long initialLength = initialResponse.Value.ContentLength;
long totalLength = ParseRangeTotalLength(initialResponse.Value.Details.ContentRange);
if (initialLength == totalLength)
{
await CopyToAsync(
initialResponse,
destination,
cancellationToken)
.ConfigureAwait(false);
return(initialResponse.GetRawResponse());
}
// Capture the etag from the first segment and construct
// conditions to ensure the blob doesn't change while we're
// downloading the remaining segments
ETag etag = initialResponse.Value.Details.ETag;
BlobRequestConditions conditionsWithEtag = CreateConditionsWithEtag(conditions, etag);
// Create a queue of tasks that will each download one segment
// of the blob. The queue maintains the order of the segments
// so we can keep appending to the end of the destination
// stream when each segment finishes.
var runningTasks = new Queue <Task <Response <BlobDownloadInfo> > >();
runningTasks.Enqueue(initialResponseTask);
// Fill the queue with tasks to download each of the remaining
// ranges in the blob
foreach (HttpRange httpRange in GetRanges(initialLength, totalLength))
{
// Add the next Task (which will start the download but
// return before it's completed downloading)
runningTasks.Enqueue(_client.DownloadAsync(
httpRange,
conditionsWithEtag,
rangeGetContentHash: false,
cancellationToken));
// If we have fewer tasks than alotted workers, then just
// continue adding tasks until we have _maxWorkerCount
// running in parallel
if (runningTasks.Count < _maxWorkerCount)
{
continue;
}
// Once all the workers are busy, wait for the first
// segment to finish downloading before we create more work
await ConsumeQueuedTask().ConfigureAwait(false);
}
// Wait for all of the remaining segments to download
while (runningTasks.Count > 0)
{
await ConsumeQueuedTask().ConfigureAwait(false);
}
return(initialResponse.GetRawResponse());
// Wait for the first segment in the queue of tasks to complete
// downloading and copy it to the destination stream
async Task ConsumeQueuedTask()
{
// Don't need to worry about 304s here because the ETag
// condition will turn into a 412 and throw a proper
// RequestFailedException
using BlobDownloadInfo result =
await runningTasks.Dequeue().ConfigureAwait(false);
// Even though the BlobDownloadInfo is returned immediately,
// CopyToAsync causes ConsumeQueuedTask to wait until the
// download is complete
await CopyToAsync(
result,
destination,
cancellationToken)
.ConfigureAwait(false);
}
}
catch (Exception ex)
{
scope.Failed(ex);
throw;
}
finally
{
scope.Dispose();
}
}
/// <summary>
/// Starts running a task responsible for receiving and processing events in the context of a specified partition.
/// </summary>
///
/// <param name="partitionId">The identifier of the Event Hub partition the task is associated with. Events will be read only from this partition.</param>
/// <param name="startingPosition">The position within the partition where the task should begin reading events.</param>
/// <param name="maximumReceiveWaitTime">The maximum amount of time to wait to for an event to be available before emitting an empty item; if <c>null</c>, empty items will not be published.</param>
/// <param name="retryOptions">The set of options to use for determining whether a failed operation should be retried and, if so, the amount of time to wait between retry attempts.</param>
/// <param name="trackLastEnqueuedEventProperties">Indicates whether or not the task should request information on the last enqueued event on the partition associated with a given event, and track that information as events are received.</param>
/// <param name="cancellationToken">An optional <see cref="CancellationToken"/> instance to signal the request to cancel the operation.</param>
///
/// <returns>The running task that is currently receiving and processing events in the context of the specified partition.</returns>
///
protected virtual Task RunPartitionProcessingAsync(string partitionId,
EventPosition startingPosition,
TimeSpan?maximumReceiveWaitTime,
EventHubsRetryOptions retryOptions,
bool trackLastEnqueuedEventProperties,
CancellationToken cancellationToken = default)
{
// TODO: should the retry options used here be the same for the abstract RetryPolicy property?
Argument.AssertNotNullOrEmpty(partitionId, nameof(partitionId));
Argument.AssertNotNull(retryOptions, nameof(retryOptions));
return(Task.Run(async() =>
{
// TODO: should we double check if a previous run already exists and close it? We have a race condition. Maybe we should throw in case another task exists.
var cancellationSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
var taskCancellationToken = cancellationSource.Token;
ActivePartitionProcessorTokenSources[partitionId] = cancellationSource;
// Context is set to default if operation fails. This shouldn't fail unless the user tries processing
// a partition they don't own.
PartitionContexts.TryGetValue(partitionId, out var context);
var clientOptions = new EventHubConsumerClientOptions
{
RetryOptions = retryOptions
};
var readOptions = new ReadEventOptions
{
MaximumWaitTime = maximumReceiveWaitTime,
TrackLastEnqueuedEventProperties = trackLastEnqueuedEventProperties
};
await using var connection = CreateConnection();
await using (var consumer = new EventHubConsumerClient(ConsumerGroup, connection, clientOptions))
{
await foreach (var partitionEvent in consumer.ReadEventsFromPartitionAsync(partitionId, startingPosition, readOptions, taskCancellationToken))
{
using DiagnosticScope diagnosticScope = EventDataInstrumentation.ClientDiagnostics.CreateScope(DiagnosticProperty.EventProcessorProcessingActivityName);
diagnosticScope.AddAttribute("kind", "server");
if (diagnosticScope.IsEnabled &&
partitionEvent.Data != null &&
EventDataInstrumentation.TryExtractDiagnosticId(partitionEvent.Data, out string diagnosticId))
{
diagnosticScope.AddLink(diagnosticId);
}
diagnosticScope.Start();
try
{
await ProcessEventAsync(partitionEvent, context).ConfigureAwait(false);
}
catch (Exception eventProcessingException)
{
diagnosticScope.Failed(eventProcessingException);
throw;
}
}
}
}));
}
public async Task <FetchResult> FetchAsync(
string dtmi, Uri repositoryUri, DependencyResolutionOption resolutionOption, CancellationToken cancellationToken = default)
{
using DiagnosticScope scope = _clientDiagnostics.CreateScope("RemoteModelFetcher.Fetch");
scope.Start();
try
{
Queue <string> work = PrepareWork(dtmi, repositoryUri, resolutionOption == DependencyResolutionOption.TryFromExpanded);
string remoteFetchError = string.Empty;
RequestFailedException requestFailedExceptionThrown = null;
Exception genericExceptionThrown = null;
while (work.Count != 0)
{
cancellationToken.ThrowIfCancellationRequested();
string tryContentPath = work.Dequeue();
ModelsRepositoryEventSource.Instance.FetchingModelContent(tryContentPath);
try
{
string content = await EvaluatePathAsync(tryContentPath, cancellationToken).ConfigureAwait(false);
return(new FetchResult()
{
Definition = content,
Path = tryContentPath
});
}
catch (RequestFailedException ex)
{
requestFailedExceptionThrown = ex;
}
catch (Exception ex)
{
genericExceptionThrown = ex;
}
if (genericExceptionThrown != null || requestFailedExceptionThrown != null)
{
remoteFetchError =
$"{string.Format(CultureInfo.InvariantCulture, StandardStrings.GenericGetModelsError, dtmi)} " +
string.Format(CultureInfo.InvariantCulture, StandardStrings.ErrorFetchingModelContent, tryContentPath);
}
}
if (requestFailedExceptionThrown != null)
{
throw new RequestFailedException(
requestFailedExceptionThrown.Status,
remoteFetchError,
requestFailedExceptionThrown.ErrorCode,
requestFailedExceptionThrown);
}
else
{
throw new RequestFailedException(
(int)HttpStatusCode.BadRequest,
remoteFetchError,
null,
genericExceptionThrown);
}
}
catch (Exception ex)
{
scope.Failed(ex);
throw;
}
}
/// <summary>
/// Attest an Intel SGX enclave.
/// </summary>
/// <param name="request">Aggregate type containing the information needed to perform an attestation operation.</param>
/// <param name="async">true if the API call should be asynchronous, false otherwise.</param>
/// <param name="cancellationToken">Cancellation token used to cancel the request.</param>
/// <returns>An <see cref="AttestationResponse{AttestationResult}"/> which contains the validated claims for the supplied <paramref name="request"/></returns>
/// <remarks>The <see cref="AttestationRequest.Evidence"/> must be an Intel SGX Quote.
/// <seealso href="https://software.intel.com/content/www/us/en/develop/articles/code-sample-intel-software-guard-extensions-remote-attestation-end-to-end-example.html"/> for more information.
///</remarks>
private async Task <AttestationResponse <AttestationResult> > AttestSgxEnclaveInternal(AttestationRequest request, bool async, CancellationToken cancellationToken = default)
{
Argument.AssertNotNull(request, nameof(request));
Argument.AssertNotNull(request.Evidence, nameof(request.Evidence));
using DiagnosticScope scope = _clientDiagnostics.CreateScope($"{nameof(AttestationClient)}.{nameof(AttestSgxEnclave)}");
scope.Start();
try
{
var attestSgxEnclaveRequest = new AttestSgxEnclaveRequest
{
Quote = request.Evidence.ToArray(),
DraftPolicyForAttestation = request.DraftPolicyForAttestation,
};
if (request.InittimeData != null)
{
attestSgxEnclaveRequest.InitTimeData = new InitTimeData
{
Data = request.InittimeData.BinaryData.ToArray(),
DataType = request.InittimeData.DataIsJson ? DataType.Json : DataType.Binary,
};
}
else
{
attestSgxEnclaveRequest.InitTimeData = null;
}
if (request.RuntimeData != null)
{
attestSgxEnclaveRequest.RuntimeData = new RuntimeData
{
Data = request.RuntimeData.BinaryData.ToArray(),
DataType = request.RuntimeData.DataIsJson ? DataType.Json : DataType.Binary,
};
}
else
{
attestSgxEnclaveRequest.RuntimeData = null;
}
Response <AttestationResponse> response;
if (async)
{
response = await _restClient.AttestSgxEnclaveAsync(attestSgxEnclaveRequest, cancellationToken).ConfigureAwait(false);
}
else
{
response = _restClient.AttestSgxEnclave(attestSgxEnclaveRequest, cancellationToken);
}
var attestationToken = AttestationToken.Deserialize(response.Value.Token, _clientDiagnostics);
if (_options.TokenOptions.ValidateToken)
{
var signers = await GetSignersAsync(async, cancellationToken).ConfigureAwait(false);
if (!await attestationToken.ValidateTokenInternal(_options.TokenOptions, signers, async, cancellationToken).ConfigureAwait(false))
{
AttestationTokenValidationFailedException.ThrowFailure(signers, attestationToken);
}
}
return(new AttestationResponse <AttestationResult>(response.GetRawResponse(), attestationToken));
}
catch (Exception ex)
{
scope.Failed(ex);
throw;
}
}
private async Task <Response <BlobContentInfo> > UploadInParallelAsync(
Stream content,
int blockSize,
BlobHttpHeaders blobHttpHeaders,
IDictionary <string, string> metadata,
BlobRequestConditions conditions,
IProgress <long> progressHandler,
AccessTier?accessTier,
CancellationToken cancellationToken)
{
// Wrap the staging and commit calls in an Upload span for
// distributed tracing
DiagnosticScope scope = _client.ClientDiagnostics.CreateScope(
_operationName ?? $"{nameof(Azure)}.{nameof(Storage)}.{nameof(Blobs)}.{nameof(BlobClient)}.{nameof(BlobClient.Upload)}");
try
{
scope.Start();
// Wrap progressHandler in a AggregatingProgressIncrementer to prevent
// progress from being reset with each stage blob operation.
if (progressHandler != null)
{
progressHandler = new AggregatingProgressIncrementer(progressHandler);
}
// The list tracking blocks IDs we're going to commit
List <string> blockIds = new List <string>();
// A list of tasks that are currently executing which will
// always be smaller than _maxWorkerCount
List <Task> runningTasks = new List <Task>();
// Partition the stream into individual blocks
await foreach (ChunkedStream block in PartitionedUploadExtensions.GetBlocksAsync(
content,
blockSize,
async: true,
_arrayPool,
cancellationToken).ConfigureAwait(false))
{
// Start staging the next block (but don't await the Task!)
string blockId = GenerateBlockId(block.AbsolutePosition);
Task task = StageBlockAsync(
block,
blockId,
conditions,
progressHandler,
cancellationToken);
// Add the block to our task and commit lists
runningTasks.Add(task);
blockIds.Add(blockId);
// If we run out of workers
if (runningTasks.Count >= _maxWorkerCount)
{
// Wait for at least one of them to finish
await Task.WhenAny(runningTasks).ConfigureAwait(false);
// Clear any completed blocks from the task list
for (int i = 0; i < runningTasks.Count; i++)
{
Task runningTask = runningTasks[i];
if (!runningTask.IsCompleted)
{
continue;
}
await runningTask.ConfigureAwait(false);
runningTasks.RemoveAt(i);
i--;
}
}
}
// Wait for all the remaining blocks to finish staging and then
// commit the block list to complete the upload
await Task.WhenAll(runningTasks).ConfigureAwait(false);
return(await _client.CommitBlockListAsync(
blockIds,
blobHttpHeaders,
metadata,
conditions,
accessTier,
cancellationToken)
.ConfigureAwait(false));
}
catch (Exception ex)
{
scope.Failed(ex);
throw;
}
finally
{
scope.Dispose();
}
}
private Response <PathInfo> UploadInSequence(
Stream content,
int blockSize,
PathHttpHeaders httpHeaders,
DataLakeRequestConditions conditions,
IProgress <long> progressHandler,
CancellationToken cancellationToken)
{
// Wrap the append and flush calls in an Upload span for
// distributed tracing
DiagnosticScope scope = _client.ClientDiagnostics.CreateScope(
_operationName ?? $"{nameof(Azure)}.{nameof(Storage)}.{nameof(Files)}.{nameof(DataLake)}.{nameof(DataLakeFileClient)}.{nameof(DataLakeFileClient.Upload)}");
try
{
scope.Start();
// Wrap progressHandler in a AggregatingProgressIncrementer to prevent
// progress from being reset with each append file operation.
if (progressHandler != null)
{
progressHandler = new AggregatingProgressIncrementer(progressHandler);
}
// Partition the stream into individual blocks and stage them
IAsyncEnumerator <ChunkedStream> enumerator =
PartitionedUploadExtensions.GetBlocksAsync(
content, blockSize, async: false, _arrayPool, cancellationToken)
.GetAsyncEnumerator(cancellationToken);
// We need to keep track of how much data we have appended to
// calculate offsets for the next appends, and the final
// position to flush
long appendedBytes = 0;
while (enumerator.MoveNextAsync().EnsureCompleted())
{
// Dispose the block after the loop iterates and return its
// memory to our ArrayPool
using ChunkedStream block = enumerator.Current;
// Append the next block
_client.Append(
new MemoryStream(block.Bytes, 0, block.Length, writable: false),
offset: appendedBytes,
leaseId: conditions?.LeaseId,
progressHandler: progressHandler,
cancellationToken: cancellationToken);
appendedBytes += block.Length;
}
// Commit the block list after everything has been staged to
// complete the upload
return(_client.Flush(
position: appendedBytes,
httpHeaders: httpHeaders,
conditions: conditions,
cancellationToken: cancellationToken));
}
catch (Exception ex)
{
scope.Failed(ex);
throw;
}
finally
{
scope.Dispose();
}
}
private async Task <Response <PathInfo> > UploadInParallelAsync(
Stream content,
int blockSize,
PathHttpHeaders httpHeaders,
DataLakeRequestConditions conditions,
IProgress <long> progressHandler,
CancellationToken cancellationToken)
{
// Wrap the staging and commit calls in an Upload span for
// distributed tracing
DiagnosticScope scope = _client.ClientDiagnostics.CreateScope(
_operationName ?? $"{nameof(Azure)}.{nameof(Storage)}.{nameof(Files)}.{nameof(DataLake)}.{nameof(DataLakeFileClient)}.{nameof(DataLakeFileClient.Upload)}");
try
{
scope.Start();
// Wrap progressHandler in a AggregatingProgressIncrementer to prevent
// progress from being reset with each stage blob operation.
if (progressHandler != null)
{
progressHandler = new AggregatingProgressIncrementer(progressHandler);
}
// A list of tasks that are currently executing which will
// always be smaller than _maxWorkerCount
List <Task> runningTasks = new List <Task>();
// We need to keep track of how much data we have appended to
// calculate offsets for the next appends, and the final
// position to flush
long appendedBytes = 0;
// Partition the stream into individual blocks
await foreach (ChunkedStream block in PartitionedUploadExtensions.GetBlocksAsync(
content, blockSize, async: true, _arrayPool, cancellationToken).ConfigureAwait(false))
{
// Start appending the next block (but don't await the Task!)
Task task = AppendBlockAsync(
block,
appendedBytes,
conditions?.LeaseId,
progressHandler,
cancellationToken);
// Add the block to our task and commit lists
runningTasks.Add(task);
appendedBytes += block.Length;
// If we run out of workers
if (runningTasks.Count >= _maxWorkerCount)
{
// Wait for at least one of them to finish
await Task.WhenAny(runningTasks).ConfigureAwait(false);
// Clear any completed blocks from the task list
for (int i = 0; i < runningTasks.Count; i++)
{
Task runningTask = runningTasks[i];
if (!runningTask.IsCompleted)
{
continue;
}
await runningTask.ConfigureAwait(false);
runningTasks.RemoveAt(i);
i--;
}
}
}
// Wait for all the remaining blocks to finish staging and then
// commit the block list to complete the upload
await Task.WhenAll(runningTasks).ConfigureAwait(false);
return(await _client.FlushAsync(
position : appendedBytes,
httpHeaders : httpHeaders,
conditions : conditions,
cancellationToken : cancellationToken)
.ConfigureAwait(false));
}
catch (Exception ex)
{
scope.Failed(ex);
throw;
}
finally
{
scope.Dispose();
}
}
private void RegisterFailed(Exception ex)
{
AzureIdentityEventSource.Singleton.GetTokenFailed(_name, _context, ex);
_scope.Failed(ex);
}